2024-11-14 21:31:51,612 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5340477f 2024-11-14 21:31:51,628 main DEBUG Took 0.013606 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 21:31:51,628 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 21:31:51,629 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 21:31:51,630 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 21:31:51,631 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:31:51,642 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 21:31:51,665 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:31:51,668 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:31:51,669 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:31:51,670 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:31:51,674 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:31:51,675 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:31:51,678 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:31:51,678 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:31:51,679 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:31:51,679 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:31:51,680 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:31:51,681 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:31:51,681 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:31:51,682 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:31:51,682 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:31:51,683 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:31:51,683 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:31:51,684 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:31:51,684 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:31:51,685 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:31:51,685 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:31:51,685 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:31:51,686 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:31:51,686 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:31:51,687 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:31:51,687 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 21:31:51,689 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:31:51,691 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 21:31:51,693 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 21:31:51,694 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 21:31:51,695 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 21:31:51,696 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 21:31:51,709 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 21:31:51,712 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 21:31:51,714 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 21:31:51,715 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 21:31:51,715 main DEBUG createAppenders(={Console}) 2024-11-14 21:31:51,716 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5340477f initialized 2024-11-14 21:31:51,717 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5340477f 2024-11-14 21:31:51,717 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5340477f OK. 2024-11-14 21:31:51,718 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 21:31:51,718 main DEBUG OutputStream closed 2024-11-14 21:31:51,718 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 21:31:51,719 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 21:31:51,719 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4659191b OK 2024-11-14 21:31:51,819 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 21:31:51,822 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 21:31:51,823 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 21:31:51,824 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 21:31:51,825 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 21:31:51,826 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 21:31:51,826 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 21:31:51,826 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 21:31:51,827 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 21:31:51,827 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 21:31:51,828 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 21:31:51,828 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 21:31:51,829 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 21:31:51,829 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 21:31:51,829 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 21:31:51,830 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 21:31:51,834 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 21:31:51,835 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 21:31:51,839 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 21:31:51,839 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@30af5b6b) with optional ClassLoader: null 2024-11-14 21:31:51,839 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 21:31:51,841 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@30af5b6b] started OK. 2024-11-14T21:31:52,258 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5 2024-11-14 21:31:52,262 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 21:31:52,262 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T21:31:52,276 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-14T21:31:52,312 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T21:31:52,331 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/cluster_a38eb657-3cf0-cdbb-aff3-937f25906809, deleteOnExit=true 2024-11-14T21:31:52,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T21:31:52,333 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/test.cache.data in system properties and HBase conf 2024-11-14T21:31:52,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T21:31:52,335 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/hadoop.log.dir in system properties and HBase conf 2024-11-14T21:31:52,335 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T21:31:52,336 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T21:31:52,336 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T21:31:52,474 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T21:31:52,632 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T21:31:52,646 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T21:31:52,647 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T21:31:52,647 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T21:31:52,648 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T21:31:52,649 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T21:31:52,649 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T21:31:52,650 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T21:31:52,650 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T21:31:52,651 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T21:31:52,652 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/nfs.dump.dir in system properties and HBase conf 2024-11-14T21:31:52,652 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/java.io.tmpdir in system properties and HBase conf 2024-11-14T21:31:52,653 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T21:31:52,653 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T21:31:52,654 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T21:31:53,955 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-14T21:31:54,111 INFO [Time-limited test {}] log.Log(170): Logging initialized @3599ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-14T21:31:54,223 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T21:31:54,319 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T21:31:54,391 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T21:31:54,392 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T21:31:54,395 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T21:31:54,426 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T21:31:54,437 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@493a7020{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/hadoop.log.dir/,AVAILABLE} 2024-11-14T21:31:54,439 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a4bbe01{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T21:31:54,754 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3654837{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/java.io.tmpdir/jetty-localhost-34541-hadoop-hdfs-3_4_1-tests_jar-_-any-10920354422088109793/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T21:31:54,774 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3edc1d8c{HTTP/1.1, (http/1.1)}{localhost:34541} 2024-11-14T21:31:54,775 INFO [Time-limited test {}] server.Server(415): Started @4263ms 2024-11-14T21:31:55,351 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T21:31:55,361 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T21:31:55,365 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T21:31:55,365 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T21:31:55,365 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T21:31:55,366 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@554622cc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/hadoop.log.dir/,AVAILABLE} 2024-11-14T21:31:55,367 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72c5887c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T21:31:55,530 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@75b88559{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/java.io.tmpdir/jetty-localhost-46255-hadoop-hdfs-3_4_1-tests_jar-_-any-1499956841326281200/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T21:31:55,531 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6e13e45b{HTTP/1.1, (http/1.1)}{localhost:46255} 2024-11-14T21:31:55,531 INFO [Time-limited test {}] server.Server(415): Started @5020ms 2024-11-14T21:31:55,587 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T21:31:56,217 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/cluster_a38eb657-3cf0-cdbb-aff3-937f25906809/data/data2/current/BP-893082529-172.17.0.2-1731619913628/current, will proceed with Du for space computation calculation, 2024-11-14T21:31:56,219 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/cluster_a38eb657-3cf0-cdbb-aff3-937f25906809/data/data1/current/BP-893082529-172.17.0.2-1731619913628/current, will proceed with Du for space computation calculation, 2024-11-14T21:31:56,270 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T21:31:56,341 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x53a17ada9f6e216 with lease ID 0xd343240736bd7fca: Processing first storage report for DS-8d96f32b-c5ce-48d5-8190-afbe21826688 from datanode DatanodeRegistration(127.0.0.1:40829, datanodeUuid=5ce7ca41-404e-436f-a55b-ea22d766e048, infoPort=45029, infoSecurePort=0, ipcPort=36493, storageInfo=lv=-57;cid=testClusterID;nsid=1678826957;c=1731619913628) 2024-11-14T21:31:56,342 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x53a17ada9f6e216 with lease ID 0xd343240736bd7fca: from storage DS-8d96f32b-c5ce-48d5-8190-afbe21826688 node DatanodeRegistration(127.0.0.1:40829, datanodeUuid=5ce7ca41-404e-436f-a55b-ea22d766e048, infoPort=45029, infoSecurePort=0, ipcPort=36493, storageInfo=lv=-57;cid=testClusterID;nsid=1678826957;c=1731619913628), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-14T21:31:56,343 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x53a17ada9f6e216 with lease ID 0xd343240736bd7fca: Processing first storage report for DS-1e8f30f8-b2d1-4b98-98a5-4d14ff3cfe2f from datanode DatanodeRegistration(127.0.0.1:40829, datanodeUuid=5ce7ca41-404e-436f-a55b-ea22d766e048, infoPort=45029, infoSecurePort=0, ipcPort=36493, storageInfo=lv=-57;cid=testClusterID;nsid=1678826957;c=1731619913628) 2024-11-14T21:31:56,343 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x53a17ada9f6e216 with lease ID 0xd343240736bd7fca: from storage DS-1e8f30f8-b2d1-4b98-98a5-4d14ff3cfe2f node DatanodeRegistration(127.0.0.1:40829, datanodeUuid=5ce7ca41-404e-436f-a55b-ea22d766e048, infoPort=45029, infoSecurePort=0, ipcPort=36493, storageInfo=lv=-57;cid=testClusterID;nsid=1678826957;c=1731619913628), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T21:31:56,379 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5 2024-11-14T21:31:56,552 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/cluster_a38eb657-3cf0-cdbb-aff3-937f25906809/zookeeper_0, clientPort=61142, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/cluster_a38eb657-3cf0-cdbb-aff3-937f25906809/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/cluster_a38eb657-3cf0-cdbb-aff3-937f25906809/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T21:31:56,586 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61142 2024-11-14T21:31:56,614 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T21:31:56,618 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T21:31:56,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741825_1001 (size=7) 2024-11-14T21:31:57,325 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275 with version=8 2024-11-14T21:31:57,325 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/hbase-staging 2024-11-14T21:31:57,442 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T21:31:57,766 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ef788a4cade8:0 server-side Connection retries=45 2024-11-14T21:31:57,779 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T21:31:57,780 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T21:31:57,787 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T21:31:57,788 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T21:31:57,788 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T21:31:57,999 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T21:31:58,082 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T21:31:58,092 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T21:31:58,098 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T21:31:58,138 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2761 (auto-detected) 2024-11-14T21:31:58,142 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-14T21:31:58,174 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38255 2024-11-14T21:31:58,210 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38255 connecting to ZooKeeper ensemble=127.0.0.1:61142 2024-11-14T21:31:58,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:382550x0, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T21:31:58,291 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38255-0x100ac5a9af00000 connected 2024-11-14T21:31:58,382 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T21:31:58,385 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T21:31:58,401 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T21:31:58,407 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275, hbase.cluster.distributed=false 2024-11-14T21:31:58,445 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T21:31:58,454 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38255 2024-11-14T21:31:58,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38255 2024-11-14T21:31:58,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38255 2024-11-14T21:31:58,458 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38255 2024-11-14T21:31:58,459 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38255 2024-11-14T21:31:58,615 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ef788a4cade8:0 server-side Connection retries=45 2024-11-14T21:31:58,618 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T21:31:58,618 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T21:31:58,619 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T21:31:58,619 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T21:31:58,619 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T21:31:58,623 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T21:31:58,626 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T21:31:58,628 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37683 2024-11-14T21:31:58,630 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37683 connecting to ZooKeeper ensemble=127.0.0.1:61142 2024-11-14T21:31:58,632 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T21:31:58,636 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T21:31:58,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:376830x0, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T21:31:58,653 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37683-0x100ac5a9af00001 connected 2024-11-14T21:31:58,654 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37683-0x100ac5a9af00001, quorum=127.0.0.1:61142, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T21:31:58,659 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T21:31:58,672 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T21:31:58,679 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37683-0x100ac5a9af00001, quorum=127.0.0.1:61142, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T21:31:58,697 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37683-0x100ac5a9af00001, quorum=127.0.0.1:61142, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T21:31:58,698 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37683 2024-11-14T21:31:58,702 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37683 2024-11-14T21:31:58,703 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37683 2024-11-14T21:31:58,710 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37683 2024-11-14T21:31:58,714 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37683 2024-11-14T21:31:58,738 DEBUG [M:0;ef788a4cade8:38255 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ef788a4cade8:38255 2024-11-14T21:31:58,739 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ef788a4cade8,38255,1731619917503 2024-11-14T21:31:58,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T21:31:58,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37683-0x100ac5a9af00001, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T21:31:58,768 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ef788a4cade8,38255,1731619917503 2024-11-14T21:31:58,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:31:58,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37683-0x100ac5a9af00001, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T21:31:58,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37683-0x100ac5a9af00001, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:31:58,810 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T21:31:58,817 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ef788a4cade8,38255,1731619917503 from backup master directory 2024-11-14T21:31:58,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ef788a4cade8,38255,1731619917503 2024-11-14T21:31:58,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T21:31:58,829 WARN [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T21:31:58,829 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ef788a4cade8,38255,1731619917503 2024-11-14T21:31:58,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37683-0x100ac5a9af00001, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T21:31:58,832 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T21:31:58,843 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T21:31:58,936 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/hbase.id] with ID: 89c9f362-26c9-422a-b55f-f256516e5794 2024-11-14T21:31:58,937 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/.tmp/hbase.id 2024-11-14T21:31:58,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741826_1002 (size=42) 2024-11-14T21:31:58,966 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/.tmp/hbase.id]:[hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/hbase.id] 2024-11-14T21:31:59,040 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T21:31:59,047 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T21:31:59,075 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 25ms. 2024-11-14T21:31:59,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:31:59,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37683-0x100ac5a9af00001, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:31:59,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741827_1003 (size=196) 2024-11-14T21:31:59,121 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T21:31:59,124 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T21:31:59,148 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:31:59,155 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:31:59,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741828_1004 (size=1189) 2024-11-14T21:31:59,625 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/data/master/store 2024-11-14T21:31:59,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741829_1005 (size=34) 2024-11-14T21:32:00,069 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T21:32:00,073 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:32:00,075 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T21:32:00,076 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T21:32:00,076 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T21:32:00,078 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T21:32:00,078 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T21:32:00,079 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T21:32:00,080 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731619920075Disabling compacts and flushes for region at 1731619920075Disabling writes for close at 1731619920078 (+3 ms)Writing region close event to WAL at 1731619920079 (+1 ms)Closed at 1731619920079 2024-11-14T21:32:00,084 WARN [master/ef788a4cade8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/data/master/store/.initializing 2024-11-14T21:32:00,084 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/WALs/ef788a4cade8,38255,1731619917503 2024-11-14T21:32:00,096 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T21:32:00,118 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ef788a4cade8%2C38255%2C1731619917503, suffix=, logDir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/WALs/ef788a4cade8,38255,1731619917503, archiveDir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/oldWALs, maxLogs=10 2024-11-14T21:32:00,153 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/WALs/ef788a4cade8,38255,1731619917503/ef788a4cade8%2C38255%2C1731619917503.1731619920124, exclude list is [], retry=0 2024-11-14T21:32:00,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40829,DS-8d96f32b-c5ce-48d5-8190-afbe21826688,DISK] 2024-11-14T21:32:00,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-14T21:32:00,240 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/WALs/ef788a4cade8,38255,1731619917503/ef788a4cade8%2C38255%2C1731619917503.1731619920124 2024-11-14T21:32:00,242 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45029:45029)] 2024-11-14T21:32:00,243 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:32:00,244 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:32:00,250 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:32:00,253 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:32:00,305 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:32:00,346 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T21:32:00,351 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:00,354 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T21:32:00,355 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:32:00,358 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T21:32:00,358 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:00,359 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:32:00,359 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:32:00,362 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T21:32:00,363 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:00,364 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:32:00,364 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:32:00,367 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T21:32:00,367 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:00,368 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:32:00,368 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:32:00,372 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:32:00,373 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:32:00,379 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:32:00,380 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:32:00,384 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T21:32:00,390 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:32:00,395 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T21:32:00,396 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65866109, jitterRate=-0.018518492579460144}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T21:32:00,401 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731619920269Initializing all the Stores at 1731619920272 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731619920273 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731619920274 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731619920274Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731619920274Cleaning up temporary data from old regions at 1731619920380 (+106 ms)Region opened successfully at 1731619920401 (+21 ms) 2024-11-14T21:32:00,403 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T21:32:00,440 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@303cc2dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef788a4cade8/172.17.0.2:0 2024-11-14T21:32:00,479 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T21:32:00,491 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T21:32:00,491 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T21:32:00,494 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T21:32:00,496 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T21:32:00,502 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-14T21:32:00,502 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T21:32:00,539 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T21:32:00,552 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T21:32:00,555 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T21:32:00,558 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T21:32:00,561 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T21:32:00,565 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T21:32:00,568 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T21:32:00,576 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T21:32:00,586 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T21:32:00,602 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T21:32:00,606 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T21:32:00,647 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T21:32:00,649 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T21:32:00,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37683-0x100ac5a9af00001, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T21:32:00,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37683-0x100ac5a9af00001, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:32:00,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T21:32:00,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:32:00,672 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ef788a4cade8,38255,1731619917503, sessionid=0x100ac5a9af00000, setting cluster-up flag (Was=false) 2024-11-14T21:32:00,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37683-0x100ac5a9af00001, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:32:00,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:32:00,702 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T21:32:00,708 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ef788a4cade8,38255,1731619917503 2024-11-14T21:32:00,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:32:00,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37683-0x100ac5a9af00001, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:32:00,732 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T21:32:00,734 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ef788a4cade8,38255,1731619917503 2024-11-14T21:32:00,746 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T21:32:00,827 INFO [RS:0;ef788a4cade8:37683 {}] regionserver.HRegionServer(746): ClusterId : 89c9f362-26c9-422a-b55f-f256516e5794 2024-11-14T21:32:00,832 DEBUG [RS:0;ef788a4cade8:37683 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T21:32:00,841 DEBUG [RS:0;ef788a4cade8:37683 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T21:32:00,841 DEBUG [RS:0;ef788a4cade8:37683 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T21:32:00,852 DEBUG [RS:0;ef788a4cade8:37683 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T21:32:00,853 DEBUG [RS:0;ef788a4cade8:37683 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69103853, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef788a4cade8/172.17.0.2:0 2024-11-14T21:32:00,853 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T21:32:00,882 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T21:32:00,894 DEBUG [RS:0;ef788a4cade8:37683 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ef788a4cade8:37683 2024-11-14T21:32:00,896 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T21:32:00,899 INFO [RS:0;ef788a4cade8:37683 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T21:32:00,899 INFO [RS:0;ef788a4cade8:37683 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T21:32:00,900 DEBUG [RS:0;ef788a4cade8:37683 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T21:32:00,906 INFO [RS:0;ef788a4cade8:37683 {}] regionserver.HRegionServer(2659): reportForDuty to master=ef788a4cade8,38255,1731619917503 with port=37683, startcode=1731619918560 2024-11-14T21:32:00,904 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ef788a4cade8,38255,1731619917503 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T21:32:00,917 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ef788a4cade8:0, corePoolSize=5, maxPoolSize=5 2024-11-14T21:32:00,918 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ef788a4cade8:0, corePoolSize=5, maxPoolSize=5 2024-11-14T21:32:00,919 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ef788a4cade8:0, corePoolSize=5, maxPoolSize=5 2024-11-14T21:32:00,922 DEBUG [RS:0;ef788a4cade8:37683 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T21:32:00,922 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ef788a4cade8:0, corePoolSize=5, maxPoolSize=5 2024-11-14T21:32:00,922 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ef788a4cade8:0, corePoolSize=10, maxPoolSize=10 2024-11-14T21:32:00,922 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:32:00,923 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ef788a4cade8:0, corePoolSize=2, maxPoolSize=2 2024-11-14T21:32:00,923 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:32:00,933 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731619950933 2024-11-14T21:32:00,935 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T21:32:00,936 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T21:32:00,941 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T21:32:00,941 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T21:32:00,942 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T21:32:00,942 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T21:32:00,945 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T21:32:00,945 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T21:32:00,951 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T21:32:00,955 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T21:32:00,957 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T21:32:00,957 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T21:32:00,970 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:00,971 INFO [PEWorker-2 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T21:32:00,973 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T21:32:00,974 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T21:32:00,981 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ef788a4cade8:0:becomeActiveMaster-HFileCleaner.large.0-1731619920977,5,FailOnTimeoutGroup] 2024-11-14T21:32:00,982 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ef788a4cade8:0:becomeActiveMaster-HFileCleaner.small.0-1731619920981,5,FailOnTimeoutGroup] 2024-11-14T21:32:00,982 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T21:32:00,983 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T21:32:00,985 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T21:32:00,987 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T21:32:00,988 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35079, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T21:32:01,000 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38255 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ef788a4cade8,37683,1731619918560 2024-11-14T21:32:01,004 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38255 {}] master.ServerManager(517): Registering regionserver=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:01,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741831_1007 (size=1321) 2024-11-14T21:32:01,018 INFO [PEWorker-2 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T21:32:01,018 INFO [PEWorker-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275 2024-11-14T21:32:01,031 DEBUG [RS:0;ef788a4cade8:37683 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275 2024-11-14T21:32:01,031 DEBUG [RS:0;ef788a4cade8:37683 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42139 2024-11-14T21:32:01,031 DEBUG [RS:0;ef788a4cade8:37683 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T21:32:01,042 DEBUG [RS:0;ef788a4cade8:37683 {}] zookeeper.ZKUtil(111): regionserver:37683-0x100ac5a9af00001, quorum=127.0.0.1:61142, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ef788a4cade8,37683,1731619918560 2024-11-14T21:32:01,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T21:32:01,043 WARN [RS:0;ef788a4cade8:37683 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T21:32:01,043 INFO [RS:0;ef788a4cade8:37683 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:32:01,043 DEBUG [RS:0;ef788a4cade8:37683 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/WALs/ef788a4cade8,37683,1731619918560 2024-11-14T21:32:01,051 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ef788a4cade8,37683,1731619918560] 2024-11-14T21:32:01,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741832_1008 (size=32) 2024-11-14T21:32:01,055 DEBUG [PEWorker-2 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:32:01,059 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T21:32:01,062 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T21:32:01,063 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:01,064 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T21:32:01,064 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T21:32:01,069 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T21:32:01,069 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:01,071 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T21:32:01,071 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T21:32:01,076 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T21:32:01,076 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:01,077 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T21:32:01,078 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T21:32:01,081 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T21:32:01,081 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:01,081 INFO [RS:0;ef788a4cade8:37683 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T21:32:01,082 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T21:32:01,082 DEBUG [PEWorker-2 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T21:32:01,084 DEBUG [PEWorker-2 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740 2024-11-14T21:32:01,085 DEBUG [PEWorker-2 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740 2024-11-14T21:32:01,096 DEBUG [PEWorker-2 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T21:32:01,096 DEBUG [PEWorker-2 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T21:32:01,098 DEBUG [PEWorker-2 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T21:32:01,100 INFO [RS:0;ef788a4cade8:37683 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T21:32:01,101 DEBUG [PEWorker-2 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T21:32:01,106 DEBUG [PEWorker-2 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T21:32:01,107 INFO [PEWorker-2 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64561695, jitterRate=-0.03795577585697174}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T21:32:01,108 INFO [RS:0;ef788a4cade8:37683 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T21:32:01,108 INFO [RS:0;ef788a4cade8:37683 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T21:32:01,109 DEBUG [PEWorker-2 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731619921056Initializing all the Stores at 1731619921059 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731619921059Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731619921059Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731619921059Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731619921059Cleaning up temporary data from old regions at 1731619921097 (+38 ms)Region opened successfully at 1731619921109 (+12 ms) 2024-11-14T21:32:01,109 DEBUG [PEWorker-2 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T21:32:01,109 INFO [PEWorker-2 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T21:32:01,109 DEBUG [PEWorker-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T21:32:01,109 DEBUG [PEWorker-2 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T21:32:01,110 DEBUG [PEWorker-2 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T21:32:01,111 INFO [RS:0;ef788a4cade8:37683 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T21:32:01,111 INFO [PEWorker-2 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T21:32:01,111 DEBUG [PEWorker-2 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731619921109Disabling compacts and flushes for region at 1731619921109Disabling writes for close at 1731619921109Writing region close event to WAL at 1731619921111 (+2 ms)Closed at 1731619921111 2024-11-14T21:32:01,116 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T21:32:01,116 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T21:32:01,120 INFO [RS:0;ef788a4cade8:37683 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T21:32:01,122 INFO [RS:0;ef788a4cade8:37683 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T21:32:01,123 DEBUG [RS:0;ef788a4cade8:37683 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:32:01,123 DEBUG [RS:0;ef788a4cade8:37683 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:32:01,123 DEBUG [RS:0;ef788a4cade8:37683 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:32:01,123 DEBUG [RS:0;ef788a4cade8:37683 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:32:01,124 DEBUG [RS:0;ef788a4cade8:37683 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:32:01,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T21:32:01,124 DEBUG [RS:0;ef788a4cade8:37683 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0, corePoolSize=2, maxPoolSize=2 2024-11-14T21:32:01,124 DEBUG [RS:0;ef788a4cade8:37683 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:32:01,124 DEBUG [RS:0;ef788a4cade8:37683 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:32:01,124 DEBUG [RS:0;ef788a4cade8:37683 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:32:01,124 DEBUG [RS:0;ef788a4cade8:37683 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:32:01,125 DEBUG [RS:0;ef788a4cade8:37683 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:32:01,125 DEBUG [RS:0;ef788a4cade8:37683 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:32:01,125 DEBUG [RS:0;ef788a4cade8:37683 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ef788a4cade8:0, corePoolSize=3, maxPoolSize=3 2024-11-14T21:32:01,125 DEBUG [RS:0;ef788a4cade8:37683 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0, corePoolSize=3, maxPoolSize=3 2024-11-14T21:32:01,127 INFO [RS:0;ef788a4cade8:37683 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T21:32:01,127 INFO [RS:0;ef788a4cade8:37683 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T21:32:01,127 INFO [RS:0;ef788a4cade8:37683 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T21:32:01,127 INFO [RS:0;ef788a4cade8:37683 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T21:32:01,127 INFO [RS:0;ef788a4cade8:37683 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T21:32:01,128 INFO [RS:0;ef788a4cade8:37683 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef788a4cade8,37683,1731619918560-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T21:32:01,136 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T21:32:01,141 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T21:32:01,164 INFO [RS:0;ef788a4cade8:37683 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T21:32:01,166 INFO [RS:0;ef788a4cade8:37683 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef788a4cade8,37683,1731619918560-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T21:32:01,167 INFO [RS:0;ef788a4cade8:37683 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T21:32:01,167 INFO [RS:0;ef788a4cade8:37683 {}] regionserver.Replication(171): ef788a4cade8,37683,1731619918560 started 2024-11-14T21:32:01,196 INFO [RS:0;ef788a4cade8:37683 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T21:32:01,197 INFO [RS:0;ef788a4cade8:37683 {}] regionserver.HRegionServer(1482): Serving as ef788a4cade8,37683,1731619918560, RpcServer on ef788a4cade8/172.17.0.2:37683, sessionid=0x100ac5a9af00001 2024-11-14T21:32:01,198 DEBUG [RS:0;ef788a4cade8:37683 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T21:32:01,198 DEBUG [RS:0;ef788a4cade8:37683 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ef788a4cade8,37683,1731619918560 2024-11-14T21:32:01,198 DEBUG [RS:0;ef788a4cade8:37683 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef788a4cade8,37683,1731619918560' 2024-11-14T21:32:01,198 DEBUG [RS:0;ef788a4cade8:37683 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T21:32:01,201 DEBUG [RS:0;ef788a4cade8:37683 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T21:32:01,202 DEBUG [RS:0;ef788a4cade8:37683 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T21:32:01,202 DEBUG [RS:0;ef788a4cade8:37683 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T21:32:01,202 DEBUG [RS:0;ef788a4cade8:37683 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ef788a4cade8,37683,1731619918560 2024-11-14T21:32:01,203 DEBUG [RS:0;ef788a4cade8:37683 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef788a4cade8,37683,1731619918560' 2024-11-14T21:32:01,203 DEBUG [RS:0;ef788a4cade8:37683 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T21:32:01,204 DEBUG [RS:0;ef788a4cade8:37683 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T21:32:01,207 DEBUG [RS:0;ef788a4cade8:37683 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T21:32:01,207 INFO [RS:0;ef788a4cade8:37683 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T21:32:01,207 INFO [RS:0;ef788a4cade8:37683 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T21:32:01,293 WARN [ef788a4cade8:38255 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-14T21:32:01,314 INFO [RS:0;ef788a4cade8:37683 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T21:32:01,318 INFO [RS:0;ef788a4cade8:37683 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ef788a4cade8%2C37683%2C1731619918560, suffix=, logDir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/WALs/ef788a4cade8,37683,1731619918560, archiveDir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/oldWALs, maxLogs=32 2024-11-14T21:32:01,337 DEBUG [RS:0;ef788a4cade8:37683 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/WALs/ef788a4cade8,37683,1731619918560/ef788a4cade8%2C37683%2C1731619918560.1731619921321, exclude list is [], retry=0 2024-11-14T21:32:01,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40829,DS-8d96f32b-c5ce-48d5-8190-afbe21826688,DISK] 2024-11-14T21:32:01,348 INFO [RS:0;ef788a4cade8:37683 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/WALs/ef788a4cade8,37683,1731619918560/ef788a4cade8%2C37683%2C1731619918560.1731619921321 2024-11-14T21:32:01,349 DEBUG [RS:0;ef788a4cade8:37683 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45029:45029)] 2024-11-14T21:32:01,546 DEBUG [ef788a4cade8:38255 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-14T21:32:01,560 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:01,567 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ef788a4cade8,37683,1731619918560, state=OPENING 2024-11-14T21:32:01,574 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T21:32:01,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:32:01,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37683-0x100ac5a9af00001, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:32:01,578 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T21:32:01,578 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T21:32:01,580 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T21:32:01,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ef788a4cade8,37683,1731619918560}] 2024-11-14T21:32:01,761 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T21:32:01,766 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49589, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T21:32:01,784 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T21:32:01,785 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:32:01,787 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T21:32:01,794 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ef788a4cade8%2C37683%2C1731619918560.meta, suffix=.meta, logDir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/WALs/ef788a4cade8,37683,1731619918560, archiveDir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/oldWALs, maxLogs=32 2024-11-14T21:32:01,814 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/WALs/ef788a4cade8,37683,1731619918560/ef788a4cade8%2C37683%2C1731619918560.meta.1731619921796.meta, exclude list is [], retry=0 2024-11-14T21:32:01,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40829,DS-8d96f32b-c5ce-48d5-8190-afbe21826688,DISK] 2024-11-14T21:32:01,827 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/WALs/ef788a4cade8,37683,1731619918560/ef788a4cade8%2C37683%2C1731619918560.meta.1731619921796.meta 2024-11-14T21:32:01,831 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45029:45029)] 2024-11-14T21:32:01,831 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:32:01,833 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T21:32:01,837 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T21:32:01,842 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T21:32:01,846 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T21:32:01,847 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:32:01,847 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T21:32:01,847 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T21:32:01,851 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T21:32:01,852 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T21:32:01,852 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:01,853 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T21:32:01,854 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T21:32:01,855 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T21:32:01,855 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:01,856 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T21:32:01,856 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T21:32:01,858 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T21:32:01,858 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:01,859 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T21:32:01,859 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T21:32:01,860 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T21:32:01,860 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:01,861 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T21:32:01,861 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T21:32:01,862 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740 2024-11-14T21:32:01,865 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740 2024-11-14T21:32:01,868 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T21:32:01,868 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T21:32:01,869 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T21:32:01,871 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T21:32:01,873 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61250862, jitterRate=-0.08729103207588196}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T21:32:01,873 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T21:32:01,876 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731619921847Writing region info on filesystem at 1731619921848 (+1 ms)Initializing all the Stores at 1731619921850 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731619921850Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731619921850Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731619921850Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731619921850Cleaning up temporary data from old regions at 1731619921868 (+18 ms)Running coprocessor post-open hooks at 1731619921873 (+5 ms)Region opened successfully at 1731619921875 (+2 ms) 2024-11-14T21:32:01,883 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731619921752 2024-11-14T21:32:01,896 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T21:32:01,896 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T21:32:01,898 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:01,900 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ef788a4cade8,37683,1731619918560, state=OPEN 2024-11-14T21:32:01,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T21:32:01,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37683-0x100ac5a9af00001, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T21:32:01,906 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T21:32:01,906 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T21:32:01,907 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:01,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T21:32:01,913 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ef788a4cade8,37683,1731619918560 in 325 msec 2024-11-14T21:32:01,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T21:32:01,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 792 msec 2024-11-14T21:32:01,923 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T21:32:01,923 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T21:32:01,951 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:32:01,953 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:32:01,976 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:32:01,980 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58259, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:32:02,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2330 sec 2024-11-14T21:32:02,021 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731619922021, completionTime=-1 2024-11-14T21:32:02,025 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-14T21:32:02,025 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T21:32:02,061 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-14T21:32:02,061 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731619982061 2024-11-14T21:32:02,061 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731620042061 2024-11-14T21:32:02,061 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 36 msec 2024-11-14T21:32:02,067 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef788a4cade8,38255,1731619917503-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T21:32:02,067 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef788a4cade8,38255,1731619917503-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T21:32:02,067 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef788a4cade8,38255,1731619917503-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T21:32:02,070 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ef788a4cade8:38255, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T21:32:02,070 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T21:32:02,075 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T21:32:02,080 DEBUG [master/ef788a4cade8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T21:32:02,111 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.282sec 2024-11-14T21:32:02,112 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T21:32:02,113 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T21:32:02,114 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T21:32:02,115 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T21:32:02,115 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T21:32:02,116 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef788a4cade8,38255,1731619917503-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T21:32:02,116 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef788a4cade8,38255,1731619917503-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T21:32:02,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34dcfc08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:02,142 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T21:32:02,144 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T21:32:02,144 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T21:32:02,144 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T21:32:02,148 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:32:02,150 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef788a4cade8,38255,1731619917503-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T21:32:02,156 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:32:02,169 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:32:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:32:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:32:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3df85c5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:32:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:32:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:02,181 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:32:02,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b845204, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:02,185 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:32:02,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:32:02,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:32:02,205 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:32:02,232 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ef788a4cade8,38255,1731619917503 2024-11-14T21:32:02,251 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=681, ProcessCount=11, AvailableMemoryMB=1419 2024-11-14T21:32:02,262 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T21:32:02,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ef788a4cade8,38255,1731619917503 2024-11-14T21:32:02,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4f9587a2 2024-11-14T21:32:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T21:32:02,285 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T21:32:02,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T21:32:02,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T21:32:02,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T21:32:02,308 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T21:32:02,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-14T21:32:02,311 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:02,314 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T21:32:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T21:32:02,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741835_1011 (size=1061) 2024-11-14T21:32:02,343 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275 2024-11-14T21:32:02,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741836_1012 (size=53) 2024-11-14T21:32:02,431 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:32:02,431 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e9722fc966803fe6d499d80c4851a0fa, disabling compactions & flushes 2024-11-14T21:32:02,431 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:02,431 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:02,431 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. after waiting 0 ms 2024-11-14T21:32:02,432 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:02,432 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:02,432 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e9722fc966803fe6d499d80c4851a0fa: Waiting for close lock at 1731619922431Disabling compacts and flushes for region at 1731619922431Disabling writes for close at 1731619922432 (+1 ms)Writing region close event to WAL at 1731619922432Closed at 1731619922432 2024-11-14T21:32:02,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T21:32:02,445 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T21:32:02,498 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731619922446"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731619922446"}]},"ts":"1731619922446"} 2024-11-14T21:32:02,509 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T21:32:02,516 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T21:32:02,521 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731619922517"}]},"ts":"1731619922517"} 2024-11-14T21:32:02,533 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T21:32:02,537 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e9722fc966803fe6d499d80c4851a0fa, ASSIGN}] 2024-11-14T21:32:02,569 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e9722fc966803fe6d499d80c4851a0fa, ASSIGN 2024-11-14T21:32:02,573 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e9722fc966803fe6d499d80c4851a0fa, ASSIGN; state=OFFLINE, location=ef788a4cade8,37683,1731619918560; forceNewPlan=false, retain=false 2024-11-14T21:32:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T21:32:02,724 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e9722fc966803fe6d499d80c4851a0fa, regionState=OPENING, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:02,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e9722fc966803fe6d499d80c4851a0fa, ASSIGN because future has completed 2024-11-14T21:32:02,731 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560}] 2024-11-14T21:32:02,893 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:02,894 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:32:02,894 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:02,894 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:32:02,894 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:02,895 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:02,897 INFO [StoreOpener-e9722fc966803fe6d499d80c4851a0fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:02,900 INFO [StoreOpener-e9722fc966803fe6d499d80c4851a0fa-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:32:02,901 INFO [StoreOpener-e9722fc966803fe6d499d80c4851a0fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9722fc966803fe6d499d80c4851a0fa columnFamilyName A 2024-11-14T21:32:02,901 DEBUG [StoreOpener-e9722fc966803fe6d499d80c4851a0fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:02,902 INFO [StoreOpener-e9722fc966803fe6d499d80c4851a0fa-1 {}] regionserver.HStore(327): Store=e9722fc966803fe6d499d80c4851a0fa/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:32:02,902 INFO [StoreOpener-e9722fc966803fe6d499d80c4851a0fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:02,904 INFO [StoreOpener-e9722fc966803fe6d499d80c4851a0fa-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:32:02,904 INFO [StoreOpener-e9722fc966803fe6d499d80c4851a0fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9722fc966803fe6d499d80c4851a0fa columnFamilyName B 2024-11-14T21:32:02,904 DEBUG [StoreOpener-e9722fc966803fe6d499d80c4851a0fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:02,905 INFO [StoreOpener-e9722fc966803fe6d499d80c4851a0fa-1 {}] regionserver.HStore(327): Store=e9722fc966803fe6d499d80c4851a0fa/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:32:02,905 INFO [StoreOpener-e9722fc966803fe6d499d80c4851a0fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:02,907 INFO [StoreOpener-e9722fc966803fe6d499d80c4851a0fa-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:32:02,907 INFO [StoreOpener-e9722fc966803fe6d499d80c4851a0fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9722fc966803fe6d499d80c4851a0fa columnFamilyName C 2024-11-14T21:32:02,908 DEBUG [StoreOpener-e9722fc966803fe6d499d80c4851a0fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:02,908 INFO [StoreOpener-e9722fc966803fe6d499d80c4851a0fa-1 {}] regionserver.HStore(327): Store=e9722fc966803fe6d499d80c4851a0fa/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:32:02,909 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:02,909 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:02,911 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:02,911 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:02,913 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:02,913 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:02,914 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T21:32:02,916 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:02,920 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T21:32:02,921 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e9722fc966803fe6d499d80c4851a0fa; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75021358, jitterRate=0.11790534853935242}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T21:32:02,921 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:02,922 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e9722fc966803fe6d499d80c4851a0fa: Running coprocessor pre-open hook at 1731619922895Writing region info on filesystem at 1731619922895Initializing all the Stores at 1731619922896 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731619922896Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731619922897 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731619922897Cleaning up temporary data from old regions at 1731619922913 (+16 ms)Running coprocessor post-open hooks at 1731619922921 (+8 ms)Region opened successfully at 1731619922922 (+1 ms) 2024-11-14T21:32:02,924 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., pid=6, masterSystemTime=1731619922886 2024-11-14T21:32:02,928 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:02,928 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:02,930 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e9722fc966803fe6d499d80c4851a0fa, regionState=OPEN, openSeqNum=2, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:02,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 because future has completed 2024-11-14T21:32:02,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T21:32:02,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 in 208 msec 2024-11-14T21:32:02,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T21:32:02,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e9722fc966803fe6d499d80c4851a0fa, ASSIGN in 405 msec 2024-11-14T21:32:02,950 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T21:32:02,950 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731619922950"}]},"ts":"1731619922950"} 2024-11-14T21:32:02,954 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T21:32:02,956 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T21:32:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T21:32:02,962 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 659 msec 2024-11-14T21:32:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T21:32:03,483 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:32:03,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c072dfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,491 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:32:03,492 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:32:03,499 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:32:03,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:32:03,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:32:03,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dd14b82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:32:03,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:32:03,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:03,508 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:32:03,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@393932c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,511 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:32:03,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:32:03,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:32:03,517 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:32:03,520 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T21:32:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is ef788a4cade8,38255,1731619917503 2024-11-14T21:32:03,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@a8f129d 2024-11-14T21:32:03,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T21:32:03,537 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T21:32:03,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5389a98d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,547 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:32:03,547 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:32:03,553 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:32:03,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:32:03,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:32:03,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b2da332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:32:03,555 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:32:03,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:03,563 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:32:03,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e3687d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@539c07a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,567 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:32:03,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:32:03,579 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:32:03,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:32:03,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:32:03,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8f5a6b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:32:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:32:03,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:03,594 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:32:03,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b383f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,599 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@484f4e79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,599 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:32:03,599 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:32:03,601 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:32:03,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:32:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:32:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30bed14c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:32:03,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:32:03,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:03,609 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:32:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1d7645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4650d4b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,619 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:32:03,619 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:32:03,630 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:32:03,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:32:03,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:32:03,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ff6f12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:32:03,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:32:03,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:03,643 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:32:03,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41f810a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22c400b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:32:03,649 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:32:03,650 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:32:03,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:32:03,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:32:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fe47336, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:32:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:32:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:03,655 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:32:03,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adf4095, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bda4713, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,658 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:32:03,658 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:32:03,659 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:32:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:32:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:32:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e154f6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:32:03,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:32:03,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:03,662 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:32:03,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2820da99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a334e4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:32:03,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:32:03,668 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:32:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:32:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:32:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3837915e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:32:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:32:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:03,671 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:32:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18548b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9e99bd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,675 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:32:03,675 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:32:03,678 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:32:03,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:32:03,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:32:03,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677effe6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:32:03,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:32:03,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:03,681 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:32:03,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@522fc2db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@482f98dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,687 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:32:03,687 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:32:03,691 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:32:03,691 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:32:03,691 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:32:03,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19dd9b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:32:03,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:32:03,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:03,699 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:32:03,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b52a890, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:03,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:32:03,711 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:32:03,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:32:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:32:03,717 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:32:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-14T21:32:03,719 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:32:03,719 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:32:03,722 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:32:03,723 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:32:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:32:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:32:03,724 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:32:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:32:03,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:32:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:32:03,731 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:32:03,732 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:32:03,732 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:32:03,733 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:32:03,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:32:03,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:32:03,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:32:03,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:32:03,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:32:03,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:32:03,738 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:32:03,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:32:03,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:32:03,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:32:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T21:32:03,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:32:03,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:32:03,744 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:32:03,746 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:32:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:32:03,748 DEBUG [Thread-145 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:32:03,749 DEBUG [Thread-145 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:32:03,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:32:03,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:32:03,752 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:32:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:32:03,755 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:32:03,755 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:32:03,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:32:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:32:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:32:03,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:32:03,772 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:32:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:32:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:32:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T21:32:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:03,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T21:32:03,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:03,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:03,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:03,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:03,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:03,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:03,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:03,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:03,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:03,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:03,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:03,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:03,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:04,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T21:32:04,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/d6928d43277c449685ed9f7047e9407a is 50, key is test_row_0/A:col10/1731619923857/Put/seqid=0 2024-11-14T21:32:04,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:04,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741837_1013 (size=12001) 2024-11-14T21:32:04,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619984049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:04,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:04,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36090 deadline: 1731619984073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:04,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731619923987, max timestamp=1731619924072Num uniques 859; 2024-11-14T21:32:04,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:32:04,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731619924038, max timestamp=1731619924070Num uniques 859; 2024-11-14T21:32:04,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T21:32:04,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731619923987, max timestamp=1731619924072Num uniques 859; 2024-11-14T21:32:04,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:32:04,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:04,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:04,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:04,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:04,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:04,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:04,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:04,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:04,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:04,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619984265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36090 deadline: 1731619984266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:04,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:04,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:04,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:04,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731619984279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:04,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619984278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:04,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:04,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619984298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:04,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:04,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:04,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:04,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:04,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:04,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:04,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:04,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:04,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T21:32:04,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:04,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619984411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:04,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:04,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731619984412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:04,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:04,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:04,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619984412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:04,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:04,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619984431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:04,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:04,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:04,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:04,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:04,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:04,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:04,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:04,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/d6928d43277c449685ed9f7047e9407a 2024-11-14T21:32:04,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:04,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36090 deadline: 1731619984511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:04,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:04,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:04,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619984631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:04,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:04,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:04,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731619984631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:04,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:04,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:04,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619984641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:04,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:04,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:04,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:04,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:04,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:04,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:04,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:04,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/d3b04dc2121d416db88778eee6313f64 is 50, key is test_row_0/B:col10/1731619923857/Put/seqid=0 2024-11-14T21:32:04,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:04,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619984683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:04,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741838_1014 (size=12001) 2024-11-14T21:32:04,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:04,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:04,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:04,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:04,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:04,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:04,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:04,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36090 deadline: 1731619984838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:04,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:04,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T21:32:04,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:04,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619984947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:04,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:04,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:04,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731619984955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:04,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:04,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:04,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619984961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:04,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:04,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:04,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:05,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:05,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:05,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:05,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:05,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:05,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:05,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:05,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619985001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:05,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/d3b04dc2121d416db88778eee6313f64 2024-11-14T21:32:05,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:05,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:05,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:05,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:05,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:05,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:05,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/6e6807177850408e8bc1d6c2ff772f32 is 50, key is test_row_0/C:col10/1731619923857/Put/seqid=0 2024-11-14T21:32:05,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741839_1015 (size=12001) 2024-11-14T21:32:05,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/6e6807177850408e8bc1d6c2ff772f32 2024-11-14T21:32:05,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/d6928d43277c449685ed9f7047e9407a as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/d6928d43277c449685ed9f7047e9407a 2024-11-14T21:32:05,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:05,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:05,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:05,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:05,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:05,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:05,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/d6928d43277c449685ed9f7047e9407a, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T21:32:05,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/d3b04dc2121d416db88778eee6313f64 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/d3b04dc2121d416db88778eee6313f64 2024-11-14T21:32:05,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36090 deadline: 1731619985363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:05,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:05,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/d3b04dc2121d416db88778eee6313f64, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T21:32:05,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/6e6807177850408e8bc1d6c2ff772f32 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/6e6807177850408e8bc1d6c2ff772f32 2024-11-14T21:32:05,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/6e6807177850408e8bc1d6c2ff772f32, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T21:32:05,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:05,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:05,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:05,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:05,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:05,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:05,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:05,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.31 KB/132415 for e9722fc966803fe6d499d80c4851a0fa in 1632ms, sequenceid=12, compaction requested=false 2024-11-14T21:32:05,520 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T21:32:05,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:05,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=136.02 KB heapSize=384.46 KB 2024-11-14T21:32:05,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:05,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:32:05,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:05,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:32:05,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:05,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:32:05,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:05,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/080b148d582d403595569de44bb97cf0 is 50, key is test_row_0/A:col10/1731619924273/Put/seqid=0 2024-11-14T21:32:05,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741840_1016 (size=14341) 2024-11-14T21:32:05,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619985600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731619985607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:05,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619985624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:05,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619985627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:05,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:05,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:05,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:05,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:05,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:05,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:05,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:05,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619985732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:05,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:05,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731619985743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:05,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:05,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619985762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:05,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619985772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:05,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:05,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:05,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:05,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:05,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:05,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:05,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T21:32:05,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619985962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:05,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731619985971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:05,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:05,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:05,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:05,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:05,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:05,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:05,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:05,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:06,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619986012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:06,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619986012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:06,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/080b148d582d403595569de44bb97cf0 2024-11-14T21:32:06,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/e5fbe764dd9d48afa2f70aa2b3fcc430 is 50, key is test_row_0/B:col10/1731619924273/Put/seqid=0 2024-11-14T21:32:06,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741841_1017 (size=12001) 2024-11-14T21:32:06,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/e5fbe764dd9d48afa2f70aa2b3fcc430 2024-11-14T21:32:06,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:06,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:06,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:06,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:06,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:06,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:06,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/d120c9badf9646b2bf930c103a3dd782 is 50, key is test_row_0/C:col10/1731619924273/Put/seqid=0 2024-11-14T21:32:06,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741842_1018 (size=12001) 2024-11-14T21:32:06,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731619986291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:06,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:06,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619986286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:06,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:06,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:06,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:06,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:06,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:06,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:06,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619986321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:06,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619986334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:06,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36090 deadline: 1731619986451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:06,468 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_1' in e9722fc966803fe6d499d80c4851a0fa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2391 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:06,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:06,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:06,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:06,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:06,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:06,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:06,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:06,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:06,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:06,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:06,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:06,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:06,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/d120c9badf9646b2bf930c103a3dd782 2024-11-14T21:32:06,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/080b148d582d403595569de44bb97cf0 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/080b148d582d403595569de44bb97cf0 2024-11-14T21:32:06,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/080b148d582d403595569de44bb97cf0, entries=200, sequenceid=48, filesize=14.0 K 2024-11-14T21:32:06,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/e5fbe764dd9d48afa2f70aa2b3fcc430 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/e5fbe764dd9d48afa2f70aa2b3fcc430 2024-11-14T21:32:06,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/e5fbe764dd9d48afa2f70aa2b3fcc430, entries=150, sequenceid=48, filesize=11.7 K 2024-11-14T21:32:06,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/d120c9badf9646b2bf930c103a3dd782 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/d120c9badf9646b2bf930c103a3dd782 2024-11-14T21:32:06,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:06,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:06,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:06,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:06,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:06,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:06,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:06,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731619986815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:06,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:06,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:06,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:06,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/d120c9badf9646b2bf930c103a3dd782, entries=150, sequenceid=48, filesize=11.7 K 2024-11-14T21:32:06,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~136.02 KB/139285, heapSize ~384.41 KB/393640, currentSize=46.96 KB/48090 for e9722fc966803fe6d499d80c4851a0fa in 1292ms, sequenceid=48, compaction requested=false 2024-11-14T21:32:06,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:06,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T21:32:06,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:06,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:06,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:06,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:06,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:06,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:06,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/8ab0e32a6b4c4612b2b616bb76e6cf4b is 50, key is test_row_0/A:col10/1731619926830/Put/seqid=0 2024-11-14T21:32:06,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741843_1019 (size=12001) 2024-11-14T21:32:06,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:06,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:06,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:06,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:06,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:06,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:07,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731619926973, max timestamp=1731619927028Num uniques 859; 2024-11-14T21:32:07,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T21:32:07,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731619926973, max timestamp=1731619927028Num uniques 859; 2024-11-14T21:32:07,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:32:07,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731619926973, max timestamp=1731619927028Num uniques 859; 2024-11-14T21:32:07,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T21:32:07,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:07,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:07,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:07,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:07,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:07,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:07,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619987180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:07,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:07,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619987184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:07,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:07,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619987188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:07,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:07,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:07,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:07,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:07,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:07,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:07,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:07,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:07,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619987311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:07,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:07,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619987321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:07,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:07,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619987321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:07,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:07,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/8ab0e32a6b4c4612b2b616bb76e6cf4b 2024-11-14T21:32:07,359 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T21:32:07,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/68a226d5f66f47bc932bde6f63fd649d is 50, key is test_row_0/B:col10/1731619926830/Put/seqid=0 2024-11-14T21:32:07,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741844_1020 (size=12001) 2024-11-14T21:32:07,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:07,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:07,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:07,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:07,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:07,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:07,492 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T21:32:07,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619987531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:07,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:07,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:07,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619987541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:07,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:07,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:07,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619987541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:07,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:07,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:07,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:07,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:07,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:07,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:07,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:07,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:07,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:07,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:07,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:07,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:07,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:07,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:07,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:07,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731619987831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:07,843 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_1' in e9722fc966803fe6d499d80c4851a0fa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:07,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:07,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/68a226d5f66f47bc932bde6f63fd649d 2024-11-14T21:32:07,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:07,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619987850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:07,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:07,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619987861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:07,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:07,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619987861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:07,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:07,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:07,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:07,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/0e1e307d55b74ddd9480787928d976ae is 50, key is test_row_0/C:col10/1731619926830/Put/seqid=0 2024-11-14T21:32:07,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T21:32:07,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:07,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:07,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:07,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:07,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:07,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:07,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:07,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741845_1021 (size=12001) 2024-11-14T21:32:07,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/0e1e307d55b74ddd9480787928d976ae 2024-11-14T21:32:07,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/8ab0e32a6b4c4612b2b616bb76e6cf4b as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8ab0e32a6b4c4612b2b616bb76e6cf4b 2024-11-14T21:32:07,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8ab0e32a6b4c4612b2b616bb76e6cf4b, entries=150, sequenceid=59, filesize=11.7 K 2024-11-14T21:32:07,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/68a226d5f66f47bc932bde6f63fd649d as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/68a226d5f66f47bc932bde6f63fd649d 2024-11-14T21:32:07,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/68a226d5f66f47bc932bde6f63fd649d, entries=150, sequenceid=59, filesize=11.7 K 2024-11-14T21:32:08,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/0e1e307d55b74ddd9480787928d976ae as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/0e1e307d55b74ddd9480787928d976ae 2024-11-14T21:32:08,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/0e1e307d55b74ddd9480787928d976ae, entries=150, sequenceid=59, filesize=11.7 K 2024-11-14T21:32:08,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.31 KB/132415 for e9722fc966803fe6d499d80c4851a0fa in 1183ms, sequenceid=59, compaction requested=true 2024-11-14T21:32:08,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:08,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:32:08,033 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:08,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:08,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:32:08,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:08,041 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:08,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:32:08,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:08,044 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:08,044 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:08,046 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/A is initiating minor compaction (all files) 2024-11-14T21:32:08,047 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/A in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:08,047 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/d6928d43277c449685ed9f7047e9407a, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/080b148d582d403595569de44bb97cf0, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8ab0e32a6b4c4612b2b616bb76e6cf4b] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=37.4 K 2024-11-14T21:32:08,048 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/B is initiating minor compaction (all files) 2024-11-14T21:32:08,048 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/B in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:08,048 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/d3b04dc2121d416db88778eee6313f64, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/e5fbe764dd9d48afa2f70aa2b3fcc430, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/68a226d5f66f47bc932bde6f63fd649d] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=35.2 K 2024-11-14T21:32:08,049 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6928d43277c449685ed9f7047e9407a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731619923840 2024-11-14T21:32:08,051 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting d3b04dc2121d416db88778eee6313f64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731619923840 2024-11-14T21:32:08,051 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 080b148d582d403595569de44bb97cf0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731619924265 2024-11-14T21:32:08,054 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting e5fbe764dd9d48afa2f70aa2b3fcc430, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731619924265 2024-11-14T21:32:08,054 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ab0e32a6b4c4612b2b616bb76e6cf4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731619925584 2024-11-14T21:32:08,055 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 68a226d5f66f47bc932bde6f63fd649d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731619925584 2024-11-14T21:32:08,078 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T21:32:08,078 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T21:32:08,080 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T21:32:08,080 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T21:32:08,081 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T21:32:08,081 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T21:32:08,083 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T21:32:08,083 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-14T21:32:08,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T21:32:08,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:08,097 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=129.31 KB heapSize=369.23 KB 2024-11-14T21:32:08,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:08,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:32:08,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:08,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:32:08,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:08,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:32:08,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/3192ff621f264eb7817264e7fa155244 is 50, key is test_row_0/A:col10/1731619927125/Put/seqid=0 2024-11-14T21:32:08,116 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#A#compaction#9 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:08,116 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#B#compaction#10 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:08,117 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/f145397daa3a42998267b7abed02bada is 50, key is test_row_0/B:col10/1731619926830/Put/seqid=0 2024-11-14T21:32:08,117 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/27f9dad005da4a708d91c9ff38499ca5 is 50, key is test_row_0/A:col10/1731619926830/Put/seqid=0 2024-11-14T21:32:08,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741848_1024 (size=12104) 2024-11-14T21:32:08,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741846_1022 (size=12001) 2024-11-14T21:32:08,175 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/3192ff621f264eb7817264e7fa155244 2024-11-14T21:32:08,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741847_1023 (size=12104) 2024-11-14T21:32:08,196 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/27f9dad005da4a708d91c9ff38499ca5 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/27f9dad005da4a708d91c9ff38499ca5 2024-11-14T21:32:08,206 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/f145397daa3a42998267b7abed02bada as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/f145397daa3a42998267b7abed02bada 2024-11-14T21:32:08,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/2a48496ac93449f486d74b1e821e7856 is 50, key is test_row_0/B:col10/1731619927125/Put/seqid=0 2024-11-14T21:32:08,225 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/A of e9722fc966803fe6d499d80c4851a0fa into 27f9dad005da4a708d91c9ff38499ca5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:08,225 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:08,229 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/A, priority=13, startTime=1731619928020; duration=0sec 2024-11-14T21:32:08,230 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:08,230 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:A 2024-11-14T21:32:08,231 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:08,231 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/B of e9722fc966803fe6d499d80c4851a0fa into f145397daa3a42998267b7abed02bada(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:08,231 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:08,231 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/B, priority=13, startTime=1731619928041; duration=0sec 2024-11-14T21:32:08,232 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:08,232 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:B 2024-11-14T21:32:08,245 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:08,245 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/C is initiating minor compaction (all files) 2024-11-14T21:32:08,245 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/C in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:08,245 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/6e6807177850408e8bc1d6c2ff772f32, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/d120c9badf9646b2bf930c103a3dd782, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/0e1e307d55b74ddd9480787928d976ae] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=35.2 K 2024-11-14T21:32:08,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741849_1025 (size=12001) 2024-11-14T21:32:08,248 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e6807177850408e8bc1d6c2ff772f32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731619923840 2024-11-14T21:32:08,248 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/2a48496ac93449f486d74b1e821e7856 2024-11-14T21:32:08,249 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting d120c9badf9646b2bf930c103a3dd782, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731619924265 2024-11-14T21:32:08,251 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e1e307d55b74ddd9480787928d976ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731619925584 2024-11-14T21:32:08,286 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#C#compaction#13 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:08,287 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/59910bfa2439438e8405bd12eafa67ec is 50, key is test_row_0/C:col10/1731619926830/Put/seqid=0 2024-11-14T21:32:08,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/5d5a966caf974a9587ef02aeadb09754 is 50, key is test_row_0/C:col10/1731619927125/Put/seqid=0 2024-11-14T21:32:08,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741850_1026 (size=12104) 2024-11-14T21:32:08,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741851_1027 (size=12001) 2024-11-14T21:32:08,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:08,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:08,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:08,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619988464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:08,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:08,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:08,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:08,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619988458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:08,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:08,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619988470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:08,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:08,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:08,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:08,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:08,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36090 deadline: 1731619988495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:08,497 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_1' in e9722fc966803fe6d499d80c4851a0fa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4425 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:08,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:08,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:08,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:08,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:08,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619988582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:08,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:08,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619988582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:08,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:08,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:08,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:08,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:08,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:08,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:08,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:08,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619988599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:08,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:08,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:08,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:08,758 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/5d5a966caf974a9587ef02aeadb09754 2024-11-14T21:32:08,767 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/59910bfa2439438e8405bd12eafa67ec as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/59910bfa2439438e8405bd12eafa67ec 2024-11-14T21:32:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/3192ff621f264eb7817264e7fa155244 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/3192ff621f264eb7817264e7fa155244 2024-11-14T21:32:08,793 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/C of e9722fc966803fe6d499d80c4851a0fa into 59910bfa2439438e8405bd12eafa67ec(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:08,793 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:08,793 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/C, priority=13, startTime=1731619928041; duration=0sec 2024-11-14T21:32:08,793 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:08,793 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:C 2024-11-14T21:32:08,812 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/3192ff621f264eb7817264e7fa155244, entries=150, sequenceid=95, filesize=11.7 K 2024-11-14T21:32:08,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619988805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:08,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619988811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:08,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619988821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:08,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/2a48496ac93449f486d74b1e821e7856 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/2a48496ac93449f486d74b1e821e7856 2024-11-14T21:32:08,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:08,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:08,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:08,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:08,867 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/2a48496ac93449f486d74b1e821e7856, entries=150, sequenceid=95, filesize=11.7 K 2024-11-14T21:32:08,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/5d5a966caf974a9587ef02aeadb09754 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/5d5a966caf974a9587ef02aeadb09754 2024-11-14T21:32:08,899 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/5d5a966caf974a9587ef02aeadb09754, entries=150, sequenceid=95, filesize=11.7 K 2024-11-14T21:32:08,902 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~129.31 KB/132415, heapSize ~369.18 KB/378040, currentSize=60.38 KB/61830 for e9722fc966803fe6d499d80c4851a0fa in 805ms, sequenceid=95, compaction requested=false 2024-11-14T21:32:08,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:08,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:08,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-14T21:32:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-14T21:32:08,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-14T21:32:08,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 5.1710 sec 2024-11-14T21:32:08,939 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 5.2260 sec 2024-11-14T21:32:09,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T21:32:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:09,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:09,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:09,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:09,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:09,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:09,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:09,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/95dc5bdd525a45e882160dcd5d729c20 is 50, key is test_row_0/A:col10/1731619928457/Put/seqid=0 2024-11-14T21:32:09,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741852_1028 (size=12001) 2024-11-14T21:32:09,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/95dc5bdd525a45e882160dcd5d729c20 2024-11-14T21:32:09,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/47d6a304aa71460e8ec3b89ca4183880 is 50, key is test_row_0/B:col10/1731619928457/Put/seqid=0 2024-11-14T21:32:09,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741853_1029 (size=12001) 2024-11-14T21:32:09,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/47d6a304aa71460e8ec3b89ca4183880 2024-11-14T21:32:09,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619989321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:09,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619989324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:09,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619989336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:09,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:09,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/8523ae2d2b9f43eebf5ae51c84188a53 is 50, key is test_row_0/C:col10/1731619928457/Put/seqid=0 2024-11-14T21:32:09,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741854_1030 (size=12001) 2024-11-14T21:32:09,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/8523ae2d2b9f43eebf5ae51c84188a53 2024-11-14T21:32:09,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/95dc5bdd525a45e882160dcd5d729c20 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/95dc5bdd525a45e882160dcd5d729c20 2024-11-14T21:32:09,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/95dc5bdd525a45e882160dcd5d729c20, entries=150, sequenceid=113, filesize=11.7 K 2024-11-14T21:32:09,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619989441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:09,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:09,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/47d6a304aa71460e8ec3b89ca4183880 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/47d6a304aa71460e8ec3b89ca4183880 2024-11-14T21:32:09,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619989442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:09,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:09,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/47d6a304aa71460e8ec3b89ca4183880, entries=150, sequenceid=113, filesize=11.7 K 2024-11-14T21:32:09,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/8523ae2d2b9f43eebf5ae51c84188a53 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/8523ae2d2b9f43eebf5ae51c84188a53 2024-11-14T21:32:09,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619989455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:09,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:09,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/8523ae2d2b9f43eebf5ae51c84188a53, entries=150, sequenceid=113, filesize=11.7 K 2024-11-14T21:32:09,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e9722fc966803fe6d499d80c4851a0fa in 353ms, sequenceid=113, compaction requested=true 2024-11-14T21:32:09,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:09,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:32:09,496 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:09,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:09,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:32:09,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:09,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:32:09,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:32:09,498 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:09,499 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:09,499 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/A is initiating minor compaction (all files) 2024-11-14T21:32:09,500 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/A in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:09,500 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/27f9dad005da4a708d91c9ff38499ca5, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/3192ff621f264eb7817264e7fa155244, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/95dc5bdd525a45e882160dcd5d729c20] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=35.3 K 2024-11-14T21:32:09,501 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27f9dad005da4a708d91c9ff38499ca5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731619925584 2024-11-14T21:32:09,502 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3192ff621f264eb7817264e7fa155244, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731619927125 2024-11-14T21:32:09,502 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95dc5bdd525a45e882160dcd5d729c20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731619928457 2024-11-14T21:32:09,505 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:09,505 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/B is initiating minor compaction (all files) 2024-11-14T21:32:09,505 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/B in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:09,505 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/f145397daa3a42998267b7abed02bada, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/2a48496ac93449f486d74b1e821e7856, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/47d6a304aa71460e8ec3b89ca4183880] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=35.3 K 2024-11-14T21:32:09,506 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting f145397daa3a42998267b7abed02bada, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731619925584 2024-11-14T21:32:09,507 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a48496ac93449f486d74b1e821e7856, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731619927125 2024-11-14T21:32:09,508 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 47d6a304aa71460e8ec3b89ca4183880, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731619928457 2024-11-14T21:32:09,534 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:09,536 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/b9980a2e6a4e4ac5b7b6c6553e035ca5 is 50, key is test_row_0/A:col10/1731619928457/Put/seqid=0 2024-11-14T21:32:09,540 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#B#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:09,540 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/7f59b63c26be4c2f9d8c1fc177ffdf9f is 50, key is test_row_0/B:col10/1731619928457/Put/seqid=0 2024-11-14T21:32:09,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741855_1031 (size=12207) 2024-11-14T21:32:09,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741856_1032 (size=12207) 2024-11-14T21:32:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:09,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T21:32:09,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:09,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:09,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:09,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:09,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:09,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:09,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/a3722f8e472d4852aaf285e69a7f4ebd is 50, key is test_row_0/A:col10/1731619929300/Put/seqid=0 2024-11-14T21:32:09,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741857_1033 (size=12101) 2024-11-14T21:32:09,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:09,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/a3722f8e472d4852aaf285e69a7f4ebd 2024-11-14T21:32:09,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619989763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:09,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:09,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:09,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619989785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:09,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619989784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:09,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:09,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:09,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/ad0c4b5495a74321935adc8db2065c02 is 50, key is test_row_0/B:col10/1731619929300/Put/seqid=0 2024-11-14T21:32:09,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:09,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731619989855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:09,864 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_1' in e9722fc966803fe6d499d80c4851a0fa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:09,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741858_1034 (size=12101) 2024-11-14T21:32:09,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:09,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619989911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:09,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:09,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619989911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:09,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:09,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:09,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619989914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:09,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:09,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:10,044 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/b9980a2e6a4e4ac5b7b6c6553e035ca5 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/b9980a2e6a4e4ac5b7b6c6553e035ca5 2024-11-14T21:32:10,064 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/A of e9722fc966803fe6d499d80c4851a0fa into b9980a2e6a4e4ac5b7b6c6553e035ca5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:10,064 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:10,064 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/A, priority=13, startTime=1731619929496; duration=0sec 2024-11-14T21:32:10,065 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:10,065 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:A 2024-11-14T21:32:10,066 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:10,072 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:10,072 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/C is initiating minor compaction (all files) 2024-11-14T21:32:10,072 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/C in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:10,073 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/59910bfa2439438e8405bd12eafa67ec, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/5d5a966caf974a9587ef02aeadb09754, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/8523ae2d2b9f43eebf5ae51c84188a53] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=35.3 K 2024-11-14T21:32:10,073 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59910bfa2439438e8405bd12eafa67ec, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731619925584 2024-11-14T21:32:10,075 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/7f59b63c26be4c2f9d8c1fc177ffdf9f as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/7f59b63c26be4c2f9d8c1fc177ffdf9f 2024-11-14T21:32:10,075 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d5a966caf974a9587ef02aeadb09754, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731619927125 2024-11-14T21:32:10,078 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8523ae2d2b9f43eebf5ae51c84188a53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731619928457 2024-11-14T21:32:10,097 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/B of e9722fc966803fe6d499d80c4851a0fa into 7f59b63c26be4c2f9d8c1fc177ffdf9f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:10,097 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:10,097 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/B, priority=13, startTime=1731619929496; duration=0sec 2024-11-14T21:32:10,097 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:10,097 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:B 2024-11-14T21:32:10,137 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#C#compaction#22 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:10,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:10,138 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/2cf98273e79148178341c8358c1a249e is 50, key is test_row_0/C:col10/1731619928457/Put/seqid=0 2024-11-14T21:32:10,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619990131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:10,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:10,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619990140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:10,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619990144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:10,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741859_1035 (size=12207) 2024-11-14T21:32:10,215 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/2cf98273e79148178341c8358c1a249e as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/2cf98273e79148178341c8358c1a249e 2024-11-14T21:32:10,229 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/C of e9722fc966803fe6d499d80c4851a0fa into 2cf98273e79148178341c8358c1a249e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:10,229 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:10,230 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/C, priority=13, startTime=1731619929497; duration=0sec 2024-11-14T21:32:10,230 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:10,230 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:C 2024-11-14T21:32:10,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/ad0c4b5495a74321935adc8db2065c02 2024-11-14T21:32:10,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/8c1d84db42584b8bb000633423971555 is 50, key is test_row_0/C:col10/1731619929300/Put/seqid=0 2024-11-14T21:32:10,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741860_1036 (size=12101) 2024-11-14T21:32:10,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/8c1d84db42584b8bb000633423971555 2024-11-14T21:32:10,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/a3722f8e472d4852aaf285e69a7f4ebd as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/a3722f8e472d4852aaf285e69a7f4ebd 2024-11-14T21:32:10,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/a3722f8e472d4852aaf285e69a7f4ebd, entries=150, sequenceid=131, filesize=11.8 K 2024-11-14T21:32:10,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/ad0c4b5495a74321935adc8db2065c02 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/ad0c4b5495a74321935adc8db2065c02 2024-11-14T21:32:10,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/ad0c4b5495a74321935adc8db2065c02, entries=150, sequenceid=131, filesize=11.8 K 2024-11-14T21:32:10,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/8c1d84db42584b8bb000633423971555 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/8c1d84db42584b8bb000633423971555 2024-11-14T21:32:10,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/8c1d84db42584b8bb000633423971555, entries=150, sequenceid=131, filesize=11.8 K 2024-11-14T21:32:10,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for e9722fc966803fe6d499d80c4851a0fa in 766ms, sequenceid=131, compaction requested=false 2024-11-14T21:32:10,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:10,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T21:32:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:10,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:10,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:10,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:10,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:10,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:10,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:10,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/5c98e151b921438dbfca863b565071aa is 50, key is test_row_0/A:col10/1731619930459/Put/seqid=0 2024-11-14T21:32:10,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619990544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:10,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:10,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619990555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:10,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741861_1037 (size=14541) 2024-11-14T21:32:10,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:10,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619990560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:10,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/5c98e151b921438dbfca863b565071aa 2024-11-14T21:32:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:10,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/47b42f17e242489c830057e493fe0f6d is 50, key is test_row_0/B:col10/1731619930459/Put/seqid=0 2024-11-14T21:32:10,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741862_1038 (size=12151) 2024-11-14T21:32:10,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/47b42f17e242489c830057e493fe0f6d 2024-11-14T21:32:10,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:10,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619990671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:10,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/bc79a8d2b436441ab7c6739f01c797ea is 50, key is test_row_0/C:col10/1731619930459/Put/seqid=0 2024-11-14T21:32:10,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:10,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619990691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:10,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:10,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619990694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:10,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:10,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:10,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:10,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:10,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:10,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:10,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741863_1039 (size=12151) 2024-11-14T21:32:10,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/bc79a8d2b436441ab7c6739f01c797ea 2024-11-14T21:32:10,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/5c98e151b921438dbfca863b565071aa as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/5c98e151b921438dbfca863b565071aa 2024-11-14T21:32:10,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/5c98e151b921438dbfca863b565071aa, entries=200, sequenceid=151, filesize=14.2 K 2024-11-14T21:32:10,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/47b42f17e242489c830057e493fe0f6d as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/47b42f17e242489c830057e493fe0f6d 2024-11-14T21:32:10,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/47b42f17e242489c830057e493fe0f6d, entries=150, sequenceid=151, filesize=11.9 K 2024-11-14T21:32:10,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/bc79a8d2b436441ab7c6739f01c797ea as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/bc79a8d2b436441ab7c6739f01c797ea 2024-11-14T21:32:10,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/bc79a8d2b436441ab7c6739f01c797ea, entries=150, sequenceid=151, filesize=11.9 K 2024-11-14T21:32:10,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for e9722fc966803fe6d499d80c4851a0fa in 335ms, sequenceid=151, compaction requested=true 2024-11-14T21:32:10,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:10,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:32:10,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:10,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:32:10,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:32:10,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:32:10,797 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:10,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:32:10,798 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:10,801 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38849 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:10,801 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/A is initiating minor compaction (all files) 2024-11-14T21:32:10,801 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/A in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:10,801 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/b9980a2e6a4e4ac5b7b6c6553e035ca5, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/a3722f8e472d4852aaf285e69a7f4ebd, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/5c98e151b921438dbfca863b565071aa] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=37.9 K 2024-11-14T21:32:10,803 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9980a2e6a4e4ac5b7b6c6553e035ca5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731619928457 2024-11-14T21:32:10,804 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:10,804 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/C is initiating minor compaction (all files) 2024-11-14T21:32:10,804 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/C in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:10,804 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/2cf98273e79148178341c8358c1a249e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/8c1d84db42584b8bb000633423971555, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/bc79a8d2b436441ab7c6739f01c797ea] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=35.6 K 2024-11-14T21:32:10,804 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3722f8e472d4852aaf285e69a7f4ebd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731619929300 2024-11-14T21:32:10,806 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cf98273e79148178341c8358c1a249e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731619928457 2024-11-14T21:32:10,806 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c98e151b921438dbfca863b565071aa, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731619929758 2024-11-14T21:32:10,807 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c1d84db42584b8bb000633423971555, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731619929300 2024-11-14T21:32:10,809 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting bc79a8d2b436441ab7c6739f01c797ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731619929758 2024-11-14T21:32:10,837 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#C#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:10,837 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#A#compaction#28 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:10,837 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/64a49ec24a83477986c7e6639b027f3d is 50, key is test_row_0/C:col10/1731619930459/Put/seqid=0 2024-11-14T21:32:10,838 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/cc423868c51b41f8b91046a200d838e5 is 50, key is test_row_0/A:col10/1731619930459/Put/seqid=0 2024-11-14T21:32:10,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741864_1040 (size=12459) 2024-11-14T21:32:10,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:10,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T21:32:10,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:10,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:10,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:10,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:10,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:10,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:10,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741865_1041 (size=12459) 2024-11-14T21:32:10,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/d59f5757bb60456ebd63d453d3bf8b7c is 50, key is test_row_0/A:col10/1731619930551/Put/seqid=0 2024-11-14T21:32:10,982 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/64a49ec24a83477986c7e6639b027f3d as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/64a49ec24a83477986c7e6639b027f3d 2024-11-14T21:32:11,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741866_1042 (size=12151) 2024-11-14T21:32:11,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619991032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:11,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:11,045 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/C of e9722fc966803fe6d499d80c4851a0fa into 64a49ec24a83477986c7e6639b027f3d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:11,045 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:11,045 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/C, priority=13, startTime=1731619930796; duration=0sec 2024-11-14T21:32:11,045 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:11,045 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:C 2024-11-14T21:32:11,046 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:11,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619991041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:11,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619991041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:11,048 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:11,048 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/B is initiating minor compaction (all files) 2024-11-14T21:32:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,048 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/B in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:11,049 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/7f59b63c26be4c2f9d8c1fc177ffdf9f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/ad0c4b5495a74321935adc8db2065c02, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/47b42f17e242489c830057e493fe0f6d] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=35.6 K 2024-11-14T21:32:11,050 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f59b63c26be4c2f9d8c1fc177ffdf9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731619928457 2024-11-14T21:32:11,051 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting ad0c4b5495a74321935adc8db2065c02, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731619929300 2024-11-14T21:32:11,051 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 47b42f17e242489c830057e493fe0f6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731619929758 2024-11-14T21:32:11,072 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#B#compaction#30 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:11,073 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/22c2a504272e49be81276fa62221d2a5 is 50, key is test_row_0/B:col10/1731619930459/Put/seqid=0 2024-11-14T21:32:11,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741867_1043 (size=12459) 2024-11-14T21:32:11,138 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/22c2a504272e49be81276fa62221d2a5 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/22c2a504272e49be81276fa62221d2a5 2024-11-14T21:32:11,152 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/B of e9722fc966803fe6d499d80c4851a0fa into 22c2a504272e49be81276fa62221d2a5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:11,153 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:11,153 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/B, priority=13, startTime=1731619930796; duration=0sec 2024-11-14T21:32:11,153 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:11,153 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:B 2024-11-14T21:32:11,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619991152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:11,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619991152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619991152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:11,338 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/cc423868c51b41f8b91046a200d838e5 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/cc423868c51b41f8b91046a200d838e5 2024-11-14T21:32:11,351 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/A of e9722fc966803fe6d499d80c4851a0fa into cc423868c51b41f8b91046a200d838e5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:11,351 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:11,351 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/A, priority=13, startTime=1731619930796; duration=0sec 2024-11-14T21:32:11,352 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:11,352 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:A 2024-11-14T21:32:11,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619991361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:11,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:11,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619991361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:11,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:11,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619991371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:11,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/d59f5757bb60456ebd63d453d3bf8b7c 2024-11-14T21:32:11,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/7fc04c74fafa49269fa84da6f5909fd1 is 50, key is test_row_0/B:col10/1731619930551/Put/seqid=0 2024-11-14T21:32:11,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741868_1044 (size=12151) 2024-11-14T21:32:11,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/7fc04c74fafa49269fa84da6f5909fd1 2024-11-14T21:32:11,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/a92f10e1b3bb4a309f8759047b0f8e1b is 50, key is test_row_0/C:col10/1731619930551/Put/seqid=0 2024-11-14T21:32:11,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741869_1045 (size=12151) 2024-11-14T21:32:11,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:11,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619991681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:11,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:11,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619991681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:11,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:11,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:11,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:11,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619991691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:11,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:11,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:11,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T21:32:11,964 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:32:11,965 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:32:11,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-14T21:32:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T21:32:11,973 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:32:11,979 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:32:11,979 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:32:12,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/a92f10e1b3bb4a309f8759047b0f8e1b 2024-11-14T21:32:12,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/d59f5757bb60456ebd63d453d3bf8b7c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/d59f5757bb60456ebd63d453d3bf8b7c 2024-11-14T21:32:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T21:32:12,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/d59f5757bb60456ebd63d453d3bf8b7c, entries=150, sequenceid=167, filesize=11.9 K 2024-11-14T21:32:12,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/7fc04c74fafa49269fa84da6f5909fd1 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/7fc04c74fafa49269fa84da6f5909fd1 2024-11-14T21:32:12,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/7fc04c74fafa49269fa84da6f5909fd1, entries=150, sequenceid=167, filesize=11.9 K 2024-11-14T21:32:12,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/a92f10e1b3bb4a309f8759047b0f8e1b as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/a92f10e1b3bb4a309f8759047b0f8e1b 2024-11-14T21:32:12,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/a92f10e1b3bb4a309f8759047b0f8e1b, entries=150, sequenceid=167, filesize=11.9 K 2024-11-14T21:32:12,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for e9722fc966803fe6d499d80c4851a0fa in 1224ms, sequenceid=167, compaction requested=false 2024-11-14T21:32:12,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:12,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T21:32:12,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:12,165 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T21:32:12,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:12,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:12,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:12,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:12,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:12,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:12,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/f619b3dad91b47168aa808a2c7a84b80 is 50, key is test_row_0/A:col10/1731619931038/Put/seqid=0 2024-11-14T21:32:12,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:12,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741870_1046 (size=12151) 2024-11-14T21:32:12,238 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/f619b3dad91b47168aa808a2c7a84b80 2024-11-14T21:32:12,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/ce852fbe903944ddb265b6ed4aad98f0 is 50, key is test_row_0/B:col10/1731619931038/Put/seqid=0 2024-11-14T21:32:12,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619992262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:12,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619992270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:12,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741871_1047 (size=12151) 2024-11-14T21:32:12,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T21:32:12,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619992282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:12,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:12,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:12,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:12,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619992391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:12,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619992400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:12,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619992411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:12,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36090 deadline: 1731619992512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:12,522 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_1' in e9722fc966803fe6d499d80c4851a0fa of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8450 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T21:32:12,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619992610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:12,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:12,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:12,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:12,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619992621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:12,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:12,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:12,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:12,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619992621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:12,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:12,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:12,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:12,687 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/ce852fbe903944ddb265b6ed4aad98f0 2024-11-14T21:32:12,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/5a201678e37a4eaa89a301239bfcd989 is 50, key is test_row_0/C:col10/1731619931038/Put/seqid=0 2024-11-14T21:32:12,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741872_1048 (size=12151) 2024-11-14T21:32:12,774 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/5a201678e37a4eaa89a301239bfcd989 2024-11-14T21:32:12,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/f619b3dad91b47168aa808a2c7a84b80 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/f619b3dad91b47168aa808a2c7a84b80 2024-11-14T21:32:12,797 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/f619b3dad91b47168aa808a2c7a84b80, entries=150, sequenceid=187, filesize=11.9 K 2024-11-14T21:32:12,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/ce852fbe903944ddb265b6ed4aad98f0 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/ce852fbe903944ddb265b6ed4aad98f0 2024-11-14T21:32:12,815 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/ce852fbe903944ddb265b6ed4aad98f0, entries=150, sequenceid=187, filesize=11.9 K 2024-11-14T21:32:12,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/5a201678e37a4eaa89a301239bfcd989 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/5a201678e37a4eaa89a301239bfcd989 2024-11-14T21:32:12,827 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/5a201678e37a4eaa89a301239bfcd989, entries=150, sequenceid=187, filesize=11.9 K 2024-11-14T21:32:12,828 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for e9722fc966803fe6d499d80c4851a0fa in 663ms, sequenceid=187, compaction requested=true 2024-11-14T21:32:12,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:12,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:12,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-14T21:32:12,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-14T21:32:12,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-14T21:32:12,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 855 msec 2024-11-14T21:32:12,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 874 msec 2024-11-14T21:32:12,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:12,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T21:32:12,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:12,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:12,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:12,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:12,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:12,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:12,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/3dc98141148046aaa83d775095c3f393 is 50, key is test_row_0/A:col10/1731619932251/Put/seqid=0 2024-11-14T21:32:12,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741873_1049 (size=12151) 2024-11-14T21:32:12,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/3dc98141148046aaa83d775095c3f393 2024-11-14T21:32:13,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619992995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:13,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:13,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619993009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:13,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619993009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:13,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:13,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/b24cc1a449d84a0f9ddcf7291dd8303c is 50, key is test_row_0/B:col10/1731619932251/Put/seqid=0 2024-11-14T21:32:13,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741874_1050 (size=12151) 2024-11-14T21:32:13,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T21:32:13,115 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:32:13,116 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:32:13,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-14T21:32:13,121 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:32:13,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T21:32:13,123 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:32:13,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:32:13,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:13,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:13,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619993121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:13,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619993121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:13,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:13,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:13,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619993123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:13,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:13,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:13,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T21:32:13,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T21:32:13,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:13,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:13,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:13,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:13,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:13,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:13,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619993340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:13,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:13,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619993340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:13,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:13,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619993340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:13,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:13,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T21:32:13,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:13,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:13,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:13,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:13,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:13,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T21:32:13,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/b24cc1a449d84a0f9ddcf7291dd8303c 2024-11-14T21:32:13,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/efe9a8669f624b11a27b5c47fe8f4fe3 is 50, key is test_row_0/C:col10/1731619932251/Put/seqid=0 2024-11-14T21:32:13,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741875_1051 (size=12151) 2024-11-14T21:32:13,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T21:32:13,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:13,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:13,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:13,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:13,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:13,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619993650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:13,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:13,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619993651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:13,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619993652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:13,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:13,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:13,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T21:32:13,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:13,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:13,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:13,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:13,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:13,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:13,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T21:32:13,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:13,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731619993893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:13,902 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_1' in e9722fc966803fe6d499d80c4851a0fa of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8295 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:13,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/efe9a8669f624b11a27b5c47fe8f4fe3 2024-11-14T21:32:13,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T21:32:13,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:13,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:13,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:13,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:13,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:13,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:13,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/3dc98141148046aaa83d775095c3f393 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/3dc98141148046aaa83d775095c3f393 2024-11-14T21:32:13,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/3dc98141148046aaa83d775095c3f393, entries=150, sequenceid=203, filesize=11.9 K 2024-11-14T21:32:13,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/b24cc1a449d84a0f9ddcf7291dd8303c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/b24cc1a449d84a0f9ddcf7291dd8303c 2024-11-14T21:32:13,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/b24cc1a449d84a0f9ddcf7291dd8303c, entries=150, sequenceid=203, filesize=11.9 K 2024-11-14T21:32:13,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/efe9a8669f624b11a27b5c47fe8f4fe3 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/efe9a8669f624b11a27b5c47fe8f4fe3 2024-11-14T21:32:14,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/efe9a8669f624b11a27b5c47fe8f4fe3, entries=150, sequenceid=203, filesize=11.9 K 2024-11-14T21:32:14,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for e9722fc966803fe6d499d80c4851a0fa in 1115ms, sequenceid=203, compaction requested=true 2024-11-14T21:32:14,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:14,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:32:14,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:14,040 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:32:14,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:32:14,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:14,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:32:14,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:32:14,041 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:32:14,059 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:32:14,059 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/B is initiating minor compaction (all files) 2024-11-14T21:32:14,059 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/B in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:14,059 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/22c2a504272e49be81276fa62221d2a5, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/7fc04c74fafa49269fa84da6f5909fd1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/ce852fbe903944ddb265b6ed4aad98f0, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/b24cc1a449d84a0f9ddcf7291dd8303c] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=47.8 K 2024-11-14T21:32:14,060 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:32:14,061 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/A is initiating minor compaction (all files) 2024-11-14T21:32:14,061 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/A in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:14,061 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/cc423868c51b41f8b91046a200d838e5, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/d59f5757bb60456ebd63d453d3bf8b7c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/f619b3dad91b47168aa808a2c7a84b80, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/3dc98141148046aaa83d775095c3f393] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=47.8 K 2024-11-14T21:32:14,061 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 22c2a504272e49be81276fa62221d2a5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731619929758 2024-11-14T21:32:14,064 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc423868c51b41f8b91046a200d838e5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731619929758 2024-11-14T21:32:14,068 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fc04c74fafa49269fa84da6f5909fd1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731619930541 2024-11-14T21:32:14,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T21:32:14,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:14,073 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T21:32:14,073 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting d59f5757bb60456ebd63d453d3bf8b7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731619930541 2024-11-14T21:32:14,073 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting ce852fbe903944ddb265b6ed4aad98f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731619930969 2024-11-14T21:32:14,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:14,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:14,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:14,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:14,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:14,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:14,073 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting f619b3dad91b47168aa808a2c7a84b80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731619930969 2024-11-14T21:32:14,073 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting b24cc1a449d84a0f9ddcf7291dd8303c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731619932251 2024-11-14T21:32:14,074 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dc98141148046aaa83d775095c3f393, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731619932251 2024-11-14T21:32:14,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/d531922ef80840f1985f4c44f776ac21 is 50, key is test_row_0/A:col10/1731619932934/Put/seqid=0 2024-11-14T21:32:14,114 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#A#compaction#40 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:14,117 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/c8e62df7dc924f389691e88bb33d440b is 50, key is test_row_0/A:col10/1731619932251/Put/seqid=0 2024-11-14T21:32:14,131 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#B#compaction#41 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:14,132 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/cb2b864e714c4de4a569d54e3d5f8235 is 50, key is test_row_0/B:col10/1731619932251/Put/seqid=0 2024-11-14T21:32:14,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741876_1052 (size=12151) 2024-11-14T21:32:14,143 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/d531922ef80840f1985f4c44f776ac21 2024-11-14T21:32:14,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741877_1053 (size=12595) 2024-11-14T21:32:14,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:14,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/ed84ac0ceb4845f7ab22e4a3980f1f2e is 50, key is test_row_0/B:col10/1731619932934/Put/seqid=0 2024-11-14T21:32:14,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741878_1054 (size=12595) 2024-11-14T21:32:14,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741879_1055 (size=12151) 2024-11-14T21:32:14,235 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/ed84ac0ceb4845f7ab22e4a3980f1f2e 2024-11-14T21:32:14,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619994228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:14,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/9b5b4c2a3d2f4ac6ae21dee55f78ecb1 is 50, key is test_row_0/C:col10/1731619932934/Put/seqid=0 2024-11-14T21:32:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:14,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619994241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:14,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:14,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619994228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T21:32:14,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:14,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741880_1056 (size=12151) 2024-11-14T21:32:14,310 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/9b5b4c2a3d2f4ac6ae21dee55f78ecb1 2024-11-14T21:32:14,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/d531922ef80840f1985f4c44f776ac21 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/d531922ef80840f1985f4c44f776ac21 2024-11-14T21:32:14,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619994363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:14,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:14,373 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/d531922ef80840f1985f4c44f776ac21, entries=150, sequenceid=219, filesize=11.9 K 2024-11-14T21:32:14,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619994370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:14,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:14,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619994371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:14,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:14,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:14,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/ed84ac0ceb4845f7ab22e4a3980f1f2e as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/ed84ac0ceb4845f7ab22e4a3980f1f2e 2024-11-14T21:32:14,426 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/ed84ac0ceb4845f7ab22e4a3980f1f2e, entries=150, sequenceid=219, filesize=11.9 K 2024-11-14T21:32:14,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/9b5b4c2a3d2f4ac6ae21dee55f78ecb1 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/9b5b4c2a3d2f4ac6ae21dee55f78ecb1 2024-11-14T21:32:14,454 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/9b5b4c2a3d2f4ac6ae21dee55f78ecb1, entries=150, sequenceid=219, filesize=11.9 K 2024-11-14T21:32:14,456 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for e9722fc966803fe6d499d80c4851a0fa in 384ms, sequenceid=219, compaction requested=true 2024-11-14T21:32:14,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:14,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:14,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-14T21:32:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-14T21:32:14,480 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-14T21:32:14,480 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3470 sec 2024-11-14T21:32:14,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.3650 sec 2024-11-14T21:32:14,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T21:32:14,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:14,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:14,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:14,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:14,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:14,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:14,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:14,619 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/c8e62df7dc924f389691e88bb33d440b as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/c8e62df7dc924f389691e88bb33d440b 2024-11-14T21:32:14,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/c4fef458338f42e0b9f3ea81790edf01 is 50, key is test_row_0/A:col10/1731619934228/Put/seqid=0 2024-11-14T21:32:14,643 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/cb2b864e714c4de4a569d54e3d5f8235 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/cb2b864e714c4de4a569d54e3d5f8235 2024-11-14T21:32:14,646 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/A of e9722fc966803fe6d499d80c4851a0fa into c8e62df7dc924f389691e88bb33d440b(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:14,646 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:14,646 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/A, priority=12, startTime=1731619934040; duration=0sec 2024-11-14T21:32:14,646 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:14,646 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:A 2024-11-14T21:32:14,646 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T21:32:14,652 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61063 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T21:32:14,652 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/C is initiating minor compaction (all files) 2024-11-14T21:32:14,653 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/C in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:14,653 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/64a49ec24a83477986c7e6639b027f3d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/a92f10e1b3bb4a309f8759047b0f8e1b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/5a201678e37a4eaa89a301239bfcd989, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/efe9a8669f624b11a27b5c47fe8f4fe3, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/9b5b4c2a3d2f4ac6ae21dee55f78ecb1] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=59.6 K 2024-11-14T21:32:14,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741881_1057 (size=14541) 2024-11-14T21:32:14,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/c4fef458338f42e0b9f3ea81790edf01 2024-11-14T21:32:14,659 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/B of e9722fc966803fe6d499d80c4851a0fa into cb2b864e714c4de4a569d54e3d5f8235(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:14,659 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:14,659 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/B, priority=12, startTime=1731619934040; duration=0sec 2024-11-14T21:32:14,660 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:14,660 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:B 2024-11-14T21:32:14,660 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64a49ec24a83477986c7e6639b027f3d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731619929758 2024-11-14T21:32:14,666 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting a92f10e1b3bb4a309f8759047b0f8e1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731619930541 2024-11-14T21:32:14,668 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a201678e37a4eaa89a301239bfcd989, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731619930969 2024-11-14T21:32:14,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/a3efb3ec66834a78a8c8fbd5ed563913 is 50, key is test_row_0/B:col10/1731619934228/Put/seqid=0 2024-11-14T21:32:14,674 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting efe9a8669f624b11a27b5c47fe8f4fe3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731619932251 2024-11-14T21:32:14,679 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b5b4c2a3d2f4ac6ae21dee55f78ecb1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731619932934 2024-11-14T21:32:14,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619994672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:14,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619994692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:14,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:14,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619994694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:14,714 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#C#compaction#46 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:14,715 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/6b8f1ee2e5c14e04948e524c124200b2 is 50, key is test_row_0/C:col10/1731619932934/Put/seqid=0 2024-11-14T21:32:14,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741882_1058 (size=12151) 2024-11-14T21:32:14,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/a3efb3ec66834a78a8c8fbd5ed563913 2024-11-14T21:32:14,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741883_1059 (size=12629) 2024-11-14T21:32:14,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/1833257b4e4249d8a061651e7874f9a9 is 50, key is test_row_0/C:col10/1731619934228/Put/seqid=0 2024-11-14T21:32:14,778 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/6b8f1ee2e5c14e04948e524c124200b2 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/6b8f1ee2e5c14e04948e524c124200b2 2024-11-14T21:32:14,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619994802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:14,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619994805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:14,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741884_1060 (size=12151) 2024-11-14T21:32:14,819 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/C of e9722fc966803fe6d499d80c4851a0fa into 6b8f1ee2e5c14e04948e524c124200b2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:14,819 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:14,819 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/C, priority=11, startTime=1731619934041; duration=0sec 2024-11-14T21:32:14,820 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:14,820 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:C 2024-11-14T21:32:14,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/1833257b4e4249d8a061651e7874f9a9 2024-11-14T21:32:14,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:14,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619994804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:14,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:14,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/c4fef458338f42e0b9f3ea81790edf01 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/c4fef458338f42e0b9f3ea81790edf01 2024-11-14T21:32:14,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/c4fef458338f42e0b9f3ea81790edf01, entries=200, sequenceid=237, filesize=14.2 K 2024-11-14T21:32:14,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/a3efb3ec66834a78a8c8fbd5ed563913 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/a3efb3ec66834a78a8c8fbd5ed563913 2024-11-14T21:32:14,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/a3efb3ec66834a78a8c8fbd5ed563913, entries=150, sequenceid=237, filesize=11.9 K 2024-11-14T21:32:14,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/1833257b4e4249d8a061651e7874f9a9 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/1833257b4e4249d8a061651e7874f9a9 2024-11-14T21:32:14,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/1833257b4e4249d8a061651e7874f9a9, entries=150, sequenceid=237, filesize=11.9 K 2024-11-14T21:32:14,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e9722fc966803fe6d499d80c4851a0fa in 313ms, sequenceid=237, compaction requested=true 2024-11-14T21:32:14,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:14,905 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:14,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:32:14,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:14,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:32:14,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:14,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:32:14,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:32:14,909 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:14,909 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/A is initiating minor compaction (all files) 2024-11-14T21:32:14,910 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/A in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:14,910 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/c8e62df7dc924f389691e88bb33d440b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/d531922ef80840f1985f4c44f776ac21, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/c4fef458338f42e0b9f3ea81790edf01] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=38.4 K 2024-11-14T21:32:14,910 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:14,915 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8e62df7dc924f389691e88bb33d440b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731619932251 2024-11-14T21:32:14,916 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting d531922ef80840f1985f4c44f776ac21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731619932934 2024-11-14T21:32:14,916 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4fef458338f42e0b9f3ea81790edf01, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731619934221 2024-11-14T21:32:14,918 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:14,918 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/B is initiating minor compaction (all files) 2024-11-14T21:32:14,918 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/B in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:14,918 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/cb2b864e714c4de4a569d54e3d5f8235, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/ed84ac0ceb4845f7ab22e4a3980f1f2e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/a3efb3ec66834a78a8c8fbd5ed563913] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=36.0 K 2024-11-14T21:32:14,919 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting cb2b864e714c4de4a569d54e3d5f8235, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731619932251 2024-11-14T21:32:14,920 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting ed84ac0ceb4845f7ab22e4a3980f1f2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731619932934 2024-11-14T21:32:14,921 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting a3efb3ec66834a78a8c8fbd5ed563913, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731619934221 2024-11-14T21:32:14,933 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#A#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:14,936 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/086e25e1b64a457993893e6eb253d543 is 50, key is test_row_0/A:col10/1731619934228/Put/seqid=0 2024-11-14T21:32:14,941 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#B#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:14,941 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/f70acac4a2c64ace9557c96fa8986b44 is 50, key is test_row_0/B:col10/1731619934228/Put/seqid=0 2024-11-14T21:32:14,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741885_1061 (size=12697) 2024-11-14T21:32:14,967 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/086e25e1b64a457993893e6eb253d543 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/086e25e1b64a457993893e6eb253d543 2024-11-14T21:32:14,987 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/A of e9722fc966803fe6d499d80c4851a0fa into 086e25e1b64a457993893e6eb253d543(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:14,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741886_1062 (size=12697) 2024-11-14T21:32:14,987 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:14,987 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/A, priority=13, startTime=1731619934905; duration=0sec 2024-11-14T21:32:14,987 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:14,987 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:A 2024-11-14T21:32:14,987 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-14T21:32:14,992 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T21:32:14,992 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T21:32:14,992 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. because compaction request was cancelled 2024-11-14T21:32:14,994 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:C 2024-11-14T21:32:14,996 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/f70acac4a2c64ace9557c96fa8986b44 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/f70acac4a2c64ace9557c96fa8986b44 2024-11-14T21:32:15,020 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/B of e9722fc966803fe6d499d80c4851a0fa into f70acac4a2c64ace9557c96fa8986b44(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:15,020 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:15,020 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/B, priority=13, startTime=1731619934909; duration=0sec 2024-11-14T21:32:15,020 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:15,020 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:B 2024-11-14T21:32:15,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T21:32:15,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:15,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:15,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:15,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:15,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:15,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:15,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/8c76b230914d46a7868f0b92c2c72019 is 50, key is test_row_0/A:col10/1731619935020/Put/seqid=0 2024-11-14T21:32:15,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741887_1063 (size=14591) 2024-11-14T21:32:15,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619995097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:15,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:15,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619995104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:15,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:15,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619995105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:15,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:15,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619995212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:15,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619995220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:15,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619995221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:15,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T21:32:15,275 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:32:15,276 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:32:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-14T21:32:15,283 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:32:15,285 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:32:15,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T21:32:15,285 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:32:15,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T21:32:15,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619995421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:15,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:15,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:15,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619995431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:15,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:15,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619995431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:15,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:15,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T21:32:15,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:15,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:15,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:15,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:15,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:15,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/8c76b230914d46a7868f0b92c2c72019 2024-11-14T21:32:15,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/2e4401742c814cdcac36c0fefac9618f is 50, key is test_row_0/B:col10/1731619935020/Put/seqid=0 2024-11-14T21:32:15,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741888_1064 (size=12201) 2024-11-14T21:32:15,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/2e4401742c814cdcac36c0fefac9618f 2024-11-14T21:32:15,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/2270d20a38394b4f90dbcde54c0399ac is 50, key is test_row_0/C:col10/1731619935020/Put/seqid=0 2024-11-14T21:32:15,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T21:32:15,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:15,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:15,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:15,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:15,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T21:32:15,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741889_1065 (size=12201) 2024-11-14T21:32:15,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/2270d20a38394b4f90dbcde54c0399ac 2024-11-14T21:32:15,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/8c76b230914d46a7868f0b92c2c72019 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8c76b230914d46a7868f0b92c2c72019 2024-11-14T21:32:15,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8c76b230914d46a7868f0b92c2c72019, entries=200, sequenceid=257, filesize=14.2 K 2024-11-14T21:32:15,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/2e4401742c814cdcac36c0fefac9618f as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/2e4401742c814cdcac36c0fefac9618f 2024-11-14T21:32:15,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/2e4401742c814cdcac36c0fefac9618f, entries=150, sequenceid=257, filesize=11.9 K 2024-11-14T21:32:15,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/2270d20a38394b4f90dbcde54c0399ac as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/2270d20a38394b4f90dbcde54c0399ac 2024-11-14T21:32:15,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/2270d20a38394b4f90dbcde54c0399ac, entries=150, sequenceid=257, filesize=11.9 K 2024-11-14T21:32:15,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e9722fc966803fe6d499d80c4851a0fa in 702ms, sequenceid=257, compaction requested=true 2024-11-14T21:32:15,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:15,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:32:15,726 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-14T21:32:15,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:15,726 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-14T21:32:15,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:32:15,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:15,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:32:15,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:15,731 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T21:32:15,731 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T21:32:15,731 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. because compaction request was cancelled 2024-11-14T21:32:15,732 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:A 2024-11-14T21:32:15,732 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:15,733 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T21:32:15,733 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T21:32:15,733 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36981 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:15,733 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. because compaction request was cancelled 2024-11-14T21:32:15,733 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:B 2024-11-14T21:32:15,733 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/C is initiating minor compaction (all files) 2024-11-14T21:32:15,733 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/C in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:15,733 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/6b8f1ee2e5c14e04948e524c124200b2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/1833257b4e4249d8a061651e7874f9a9, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/2270d20a38394b4f90dbcde54c0399ac] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=36.1 K 2024-11-14T21:32:15,734 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b8f1ee2e5c14e04948e524c124200b2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731619932934 2024-11-14T21:32:15,735 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1833257b4e4249d8a061651e7874f9a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731619934221 2024-11-14T21:32:15,735 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2270d20a38394b4f90dbcde54c0399ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731619934669 2024-11-14T21:32:15,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:15,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T21:32:15,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:15,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:15,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:15,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:15,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:15,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:15,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/5c74734d7023439ea9f137d0b01d8057 is 50, key is test_row_0/A:col10/1731619935736/Put/seqid=0 2024-11-14T21:32:15,768 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#C#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:15,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T21:32:15,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:15,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:15,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:15,768 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/50638068572d44f9a3192d230539c628 is 50, key is test_row_0/C:col10/1731619935020/Put/seqid=0 2024-11-14T21:32:15,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:15,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:15,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:15,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741890_1066 (size=12301) 2024-11-14T21:32:15,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741891_1067 (size=12781) 2024-11-14T21:32:15,850 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/50638068572d44f9a3192d230539c628 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/50638068572d44f9a3192d230539c628 2024-11-14T21:32:15,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619995838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:15,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619995839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:15,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:15,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619995840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:15,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:15,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:15,865 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/C of e9722fc966803fe6d499d80c4851a0fa into 50638068572d44f9a3192d230539c628(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:15,865 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:15,865 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/C, priority=13, startTime=1731619935727; duration=0sec 2024-11-14T21:32:15,866 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:15,866 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:C 2024-11-14T21:32:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T21:32:15,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T21:32:15,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:15,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:15,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:15,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:15,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:15,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:15,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619995960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:15,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619995960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:15,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:15,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619995961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:15,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:15,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:16,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T21:32:16,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:16,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:16,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:16,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:16,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:16,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:16,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619996171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:16,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619996171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:16,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:16,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:16,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:16,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619996180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:16,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:16,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:16,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:16,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/5c74734d7023439ea9f137d0b01d8057 2024-11-14T21:32:16,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/34621090067e4982bdc537c8b435d495 is 50, key is test_row_0/B:col10/1731619935736/Put/seqid=0 2024-11-14T21:32:16,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T21:32:16,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:16,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:16,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:16,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:16,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:16,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741892_1068 (size=12301) 2024-11-14T21:32:16,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T21:32:16,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:16,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:16,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:16,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:16,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T21:32:16,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619996482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:16,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619996492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:16,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619996492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:16,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:16,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T21:32:16,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:16,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:16,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:16,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:16,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:16,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/34621090067e4982bdc537c8b435d495 2024-11-14T21:32:16,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/a79c794842404f30812323be2bf22616 is 50, key is test_row_0/C:col10/1731619935736/Put/seqid=0 2024-11-14T21:32:16,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T21:32:16,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741893_1069 (size=12301) 2024-11-14T21:32:16,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:16,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:16,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:16,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:16,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:16,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/a79c794842404f30812323be2bf22616 2024-11-14T21:32:16,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:16,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/5c74734d7023439ea9f137d0b01d8057 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/5c74734d7023439ea9f137d0b01d8057 2024-11-14T21:32:16,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/5c74734d7023439ea9f137d0b01d8057, entries=150, sequenceid=275, filesize=12.0 K 2024-11-14T21:32:16,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/34621090067e4982bdc537c8b435d495 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/34621090067e4982bdc537c8b435d495 2024-11-14T21:32:16,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/34621090067e4982bdc537c8b435d495, entries=150, sequenceid=275, filesize=12.0 K 2024-11-14T21:32:16,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/a79c794842404f30812323be2bf22616 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/a79c794842404f30812323be2bf22616 2024-11-14T21:32:16,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/a79c794842404f30812323be2bf22616, entries=150, sequenceid=275, filesize=12.0 K 2024-11-14T21:32:16,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e9722fc966803fe6d499d80c4851a0fa in 1045ms, sequenceid=275, compaction requested=true 2024-11-14T21:32:16,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:16,784 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:16,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:32:16,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:16,785 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:16,785 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:16,785 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/A is initiating minor compaction (all files) 2024-11-14T21:32:16,785 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/A in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:16,785 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/086e25e1b64a457993893e6eb253d543, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8c76b230914d46a7868f0b92c2c72019, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/5c74734d7023439ea9f137d0b01d8057] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=38.7 K 2024-11-14T21:32:16,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:32:16,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:16,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:32:16,786 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 086e25e1b64a457993893e6eb253d543, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731619934221 2024-11-14T21:32:16,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:16,786 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c76b230914d46a7868f0b92c2c72019, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731619934669 2024-11-14T21:32:16,787 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c74734d7023439ea9f137d0b01d8057, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731619935091 2024-11-14T21:32:16,787 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:16,787 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/B is initiating minor compaction (all files) 2024-11-14T21:32:16,787 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/B in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:16,787 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/f70acac4a2c64ace9557c96fa8986b44, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/2e4401742c814cdcac36c0fefac9618f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/34621090067e4982bdc537c8b435d495] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=36.3 K 2024-11-14T21:32:16,788 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting f70acac4a2c64ace9557c96fa8986b44, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731619934221 2024-11-14T21:32:16,789 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e4401742c814cdcac36c0fefac9618f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731619934669 2024-11-14T21:32:16,790 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 34621090067e4982bdc537c8b435d495, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731619935091 2024-11-14T21:32:16,808 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#A#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:16,808 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/a8e27ee8d61746148fef3f6ed94fad93 is 50, key is test_row_0/A:col10/1731619935736/Put/seqid=0 2024-11-14T21:32:16,811 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#B#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:16,812 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/a90e5794810e45ef8cfa9d4a49ddeb5b is 50, key is test_row_0/B:col10/1731619935736/Put/seqid=0 2024-11-14T21:32:16,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741894_1070 (size=12949) 2024-11-14T21:32:16,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T21:32:16,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:16,888 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T21:32:16,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:16,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:16,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:16,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:16,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:16,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:16,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741895_1071 (size=12949) 2024-11-14T21:32:16,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/6ea605de5b5d41d29ddf5b31b361f335 is 50, key is test_row_0/A:col10/1731619935829/Put/seqid=0 2024-11-14T21:32:16,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741896_1072 (size=12301) 2024-11-14T21:32:16,945 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/6ea605de5b5d41d29ddf5b31b361f335 2024-11-14T21:32:16,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/5d1426e47fd9423c8268d70b3af264ee is 50, key is test_row_0/B:col10/1731619935829/Put/seqid=0 2024-11-14T21:32:16,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741897_1073 (size=12301) 2024-11-14T21:32:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:17,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:17,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619997031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:17,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:17,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619997033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:17,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619997035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:17,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:17,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:17,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:17,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619997141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619997141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:17,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:17,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:17,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619997150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:17,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:17,294 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/a8e27ee8d61746148fef3f6ed94fad93 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/a8e27ee8d61746148fef3f6ed94fad93 2024-11-14T21:32:17,304 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/A of e9722fc966803fe6d499d80c4851a0fa into a8e27ee8d61746148fef3f6ed94fad93(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:17,305 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:17,305 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/A, priority=13, startTime=1731619936784; duration=0sec 2024-11-14T21:32:17,306 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:17,306 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:A 2024-11-14T21:32:17,306 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-14T21:32:17,315 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T21:32:17,315 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T21:32:17,315 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. because compaction request was cancelled 2024-11-14T21:32:17,315 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:C 2024-11-14T21:32:17,321 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/a90e5794810e45ef8cfa9d4a49ddeb5b as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/a90e5794810e45ef8cfa9d4a49ddeb5b 2024-11-14T21:32:17,340 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/B of e9722fc966803fe6d499d80c4851a0fa into a90e5794810e45ef8cfa9d4a49ddeb5b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:17,341 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:17,341 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/B, priority=13, startTime=1731619936785; duration=0sec 2024-11-14T21:32:17,341 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:17,341 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:B 2024-11-14T21:32:17,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:17,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619997350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:17,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:17,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:17,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619997352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:17,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:17,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619997361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:17,380 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/5d1426e47fd9423c8268d70b3af264ee 2024-11-14T21:32:17,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/5a0e1eabe218454581b87afba6b58b5c is 50, key is test_row_0/C:col10/1731619935829/Put/seqid=0 2024-11-14T21:32:17,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741898_1074 (size=12301) 2024-11-14T21:32:17,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T21:32:17,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:17,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619997660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:17,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619997660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:17,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:17,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619997671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:17,817 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/5a0e1eabe218454581b87afba6b58b5c 2024-11-14T21:32:17,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/6ea605de5b5d41d29ddf5b31b361f335 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/6ea605de5b5d41d29ddf5b31b361f335 2024-11-14T21:32:17,843 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/6ea605de5b5d41d29ddf5b31b361f335, entries=150, sequenceid=290, filesize=12.0 K 2024-11-14T21:32:17,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/5d1426e47fd9423c8268d70b3af264ee as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/5d1426e47fd9423c8268d70b3af264ee 2024-11-14T21:32:17,856 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/5d1426e47fd9423c8268d70b3af264ee, entries=150, sequenceid=290, filesize=12.0 K 2024-11-14T21:32:17,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/5a0e1eabe218454581b87afba6b58b5c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/5a0e1eabe218454581b87afba6b58b5c 2024-11-14T21:32:17,866 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/5a0e1eabe218454581b87afba6b58b5c, entries=150, sequenceid=290, filesize=12.0 K 2024-11-14T21:32:17,875 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e9722fc966803fe6d499d80c4851a0fa in 987ms, sequenceid=290, compaction requested=true 2024-11-14T21:32:17,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:17,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:17,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-14T21:32:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-14T21:32:17,894 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-14T21:32:17,894 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5970 sec 2024-11-14T21:32:17,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.6220 sec 2024-11-14T21:32:18,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T21:32:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:18,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:18,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:18,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:18,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:18,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:18,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:18,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/240d2bc2589e4561a19edd8295607633 is 50, key is test_row_0/A:col10/1731619937025/Put/seqid=0 2024-11-14T21:32:18,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741899_1075 (size=14741) 2024-11-14T21:32:18,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619998238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:18,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:18,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619998247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:18,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619998248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:18,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:18,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619998351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:18,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619998363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:18,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619998366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:18,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:18,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619998561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:18,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619998570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:18,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:18,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619998580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:18,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/240d2bc2589e4561a19edd8295607633 2024-11-14T21:32:18,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/bcce44460d2c47b2b583b0d1ea58f12f is 50, key is test_row_0/B:col10/1731619937025/Put/seqid=0 2024-11-14T21:32:18,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741900_1076 (size=12301) 2024-11-14T21:32:18,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/bcce44460d2c47b2b583b0d1ea58f12f 2024-11-14T21:32:18,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/82082002cc1a4418addd47da135e88f2 is 50, key is test_row_0/C:col10/1731619937025/Put/seqid=0 2024-11-14T21:32:18,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741901_1077 (size=12301) 2024-11-14T21:32:18,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/82082002cc1a4418addd47da135e88f2 2024-11-14T21:32:18,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/240d2bc2589e4561a19edd8295607633 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/240d2bc2589e4561a19edd8295607633 2024-11-14T21:32:18,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/240d2bc2589e4561a19edd8295607633, entries=200, sequenceid=312, filesize=14.4 K 2024-11-14T21:32:18,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/bcce44460d2c47b2b583b0d1ea58f12f as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/bcce44460d2c47b2b583b0d1ea58f12f 2024-11-14T21:32:18,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/bcce44460d2c47b2b583b0d1ea58f12f, entries=150, sequenceid=312, filesize=12.0 K 2024-11-14T21:32:18,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/82082002cc1a4418addd47da135e88f2 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/82082002cc1a4418addd47da135e88f2 2024-11-14T21:32:18,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/82082002cc1a4418addd47da135e88f2, entries=150, sequenceid=312, filesize=12.0 K 2024-11-14T21:32:18,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for e9722fc966803fe6d499d80c4851a0fa in 588ms, sequenceid=312, compaction requested=true 2024-11-14T21:32:18,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:18,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:32:18,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:18,773 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:18,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:32:18,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:18,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:32:18,773 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:18,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:18,775 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:18,775 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/B is initiating minor compaction (all files) 2024-11-14T21:32:18,775 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/B in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:18,775 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/a90e5794810e45ef8cfa9d4a49ddeb5b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/5d1426e47fd9423c8268d70b3af264ee, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/bcce44460d2c47b2b583b0d1ea58f12f] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=36.7 K 2024-11-14T21:32:18,775 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:18,775 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/A is initiating minor compaction (all files) 2024-11-14T21:32:18,775 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting a90e5794810e45ef8cfa9d4a49ddeb5b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731619935091 2024-11-14T21:32:18,775 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/A in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:18,776 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/a8e27ee8d61746148fef3f6ed94fad93, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/6ea605de5b5d41d29ddf5b31b361f335, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/240d2bc2589e4561a19edd8295607633] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=39.1 K 2024-11-14T21:32:18,777 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8e27ee8d61746148fef3f6ed94fad93, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731619935091 2024-11-14T21:32:18,777 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d1426e47fd9423c8268d70b3af264ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731619935813 2024-11-14T21:32:18,777 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ea605de5b5d41d29ddf5b31b361f335, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731619935813 2024-11-14T21:32:18,777 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting bcce44460d2c47b2b583b0d1ea58f12f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731619937025 2024-11-14T21:32:18,778 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 240d2bc2589e4561a19edd8295607633, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731619937025 2024-11-14T21:32:18,805 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#A#compaction#65 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:18,805 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/8e2f91dce5784d3c819dc91080a657e2 is 50, key is test_row_0/A:col10/1731619937025/Put/seqid=0 2024-11-14T21:32:18,812 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#B#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:18,813 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/766a94b7e3214b60b837f3016946ae0a is 50, key is test_row_0/B:col10/1731619937025/Put/seqid=0 2024-11-14T21:32:18,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741903_1079 (size=13051) 2024-11-14T21:32:18,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741902_1078 (size=13051) 2024-11-14T21:32:18,864 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/8e2f91dce5784d3c819dc91080a657e2 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8e2f91dce5784d3c819dc91080a657e2 2024-11-14T21:32:18,874 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/A of e9722fc966803fe6d499d80c4851a0fa into 8e2f91dce5784d3c819dc91080a657e2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:18,874 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:18,874 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/A, priority=13, startTime=1731619938773; duration=0sec 2024-11-14T21:32:18,874 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:18,874 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:A 2024-11-14T21:32:18,874 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:32:18,876 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:32:18,876 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/C is initiating minor compaction (all files) 2024-11-14T21:32:18,876 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/C in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:18,876 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/50638068572d44f9a3192d230539c628, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/a79c794842404f30812323be2bf22616, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/5a0e1eabe218454581b87afba6b58b5c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/82082002cc1a4418addd47da135e88f2] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=48.5 K 2024-11-14T21:32:18,877 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50638068572d44f9a3192d230539c628, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731619934669 2024-11-14T21:32:18,877 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting a79c794842404f30812323be2bf22616, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731619935091 2024-11-14T21:32:18,878 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a0e1eabe218454581b87afba6b58b5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731619935813 2024-11-14T21:32:18,878 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82082002cc1a4418addd47da135e88f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731619937025 2024-11-14T21:32:18,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T21:32:18,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:18,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:18,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:18,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:18,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:18,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:18,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/e94a81d31ae04fc783a3cb296c36b11f is 50, key is test_row_0/A:col10/1731619938243/Put/seqid=0 2024-11-14T21:32:18,915 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#C#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:18,916 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/7111af08da8046bc87246c0a1eaabe6f is 50, key is test_row_0/C:col10/1731619937025/Put/seqid=0 2024-11-14T21:32:18,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741905_1081 (size=13017) 2024-11-14T21:32:18,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741904_1080 (size=14741) 2024-11-14T21:32:18,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/e94a81d31ae04fc783a3cb296c36b11f 2024-11-14T21:32:18,974 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/7111af08da8046bc87246c0a1eaabe6f as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/7111af08da8046bc87246c0a1eaabe6f 2024-11-14T21:32:18,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619998975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:18,993 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/C of e9722fc966803fe6d499d80c4851a0fa into 7111af08da8046bc87246c0a1eaabe6f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:18,993 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:18,994 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/C, priority=12, startTime=1731619938773; duration=0sec 2024-11-14T21:32:18,994 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:18,994 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:C 2024-11-14T21:32:18,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619998979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:18,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:18,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619998980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:18,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:18,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:19,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/dab55aefa72149f782a165b82b8b0214 is 50, key is test_row_0/B:col10/1731619938243/Put/seqid=0 2024-11-14T21:32:19,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741906_1082 (size=12301) 2024-11-14T21:32:19,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619999094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:19,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:19,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619999101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619999101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:19,283 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/766a94b7e3214b60b837f3016946ae0a as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/766a94b7e3214b60b837f3016946ae0a 2024-11-14T21:32:19,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:19,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619999302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:19,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:19,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:19,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619999311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:19,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619999313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:19,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:19,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:19,327 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/B of e9722fc966803fe6d499d80c4851a0fa into 766a94b7e3214b60b837f3016946ae0a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:19,327 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:19,327 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/B, priority=13, startTime=1731619938773; duration=0sec 2024-11-14T21:32:19,328 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:19,328 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:B 2024-11-14T21:32:19,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/dab55aefa72149f782a165b82b8b0214 2024-11-14T21:32:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T21:32:19,451 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:32:19,455 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:32:19,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-14T21:32:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T21:32:19,466 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:32:19,472 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:32:19,472 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:32:19,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/3879c8ca35944a1bbdd2b76756897e69 is 50, key is test_row_0/C:col10/1731619938243/Put/seqid=0 2024-11-14T21:32:19,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741907_1083 (size=12301) 2024-11-14T21:32:19,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/3879c8ca35944a1bbdd2b76756897e69 2024-11-14T21:32:19,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/e94a81d31ae04fc783a3cb296c36b11f as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/e94a81d31ae04fc783a3cb296c36b11f 2024-11-14T21:32:19,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/e94a81d31ae04fc783a3cb296c36b11f, entries=200, sequenceid=327, filesize=14.4 K 2024-11-14T21:32:19,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/dab55aefa72149f782a165b82b8b0214 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/dab55aefa72149f782a165b82b8b0214 2024-11-14T21:32:19,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/dab55aefa72149f782a165b82b8b0214, entries=150, sequenceid=327, filesize=12.0 K 2024-11-14T21:32:19,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/3879c8ca35944a1bbdd2b76756897e69 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/3879c8ca35944a1bbdd2b76756897e69 2024-11-14T21:32:19,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/3879c8ca35944a1bbdd2b76756897e69, entries=150, sequenceid=327, filesize=12.0 K 2024-11-14T21:32:19,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T21:32:19,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for e9722fc966803fe6d499d80c4851a0fa in 682ms, sequenceid=327, compaction requested=false 2024-11-14T21:32:19,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:19,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:19,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T21:32:19,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:19,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:19,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:19,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:19,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:19,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:19,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T21:32:19,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:19,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:19,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:19,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:19,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:19,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/5aebc75183874177a79e3bef71be93af is 50, key is test_row_0/A:col10/1731619938941/Put/seqid=0 2024-11-14T21:32:19,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741908_1084 (size=12301) 2024-11-14T21:32:19,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/5aebc75183874177a79e3bef71be93af 2024-11-14T21:32:19,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:19,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619999669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:19,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:19,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:19,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619999670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:19,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:19,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619999685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:19,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/b9a9bbe00a6a42fab966c2f2ec5663e4 is 50, key is test_row_0/B:col10/1731619938941/Put/seqid=0 2024-11-14T21:32:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T21:32:19,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T21:32:19,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:19,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:19,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:19,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:19,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:19,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:19,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741909_1085 (size=12301) 2024-11-14T21:32:19,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731619999792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:19,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:19,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:19,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731619999803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:19,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/b9a9bbe00a6a42fab966c2f2ec5663e4 2024-11-14T21:32:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:19,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:19,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731619999814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:19,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:19,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:19,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/e7ba4822ee0f498cb12705d247a42175 is 50, key is test_row_0/C:col10/1731619938941/Put/seqid=0 2024-11-14T21:32:19,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741910_1086 (size=12301) 2024-11-14T21:32:19,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T21:32:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:19,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:20,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:20,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731620000013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:20,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:20,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:20,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:20,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:20,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731620000022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:20,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:20,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:20,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:20,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:20,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731620000031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:20,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:20,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:20,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T21:32:20,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T21:32:20,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:20,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:20,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:20,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:20,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:20,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T21:32:20,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:20,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:20,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:20,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:20,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:20,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/e7ba4822ee0f498cb12705d247a42175 2024-11-14T21:32:20,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:20,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/5aebc75183874177a79e3bef71be93af as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/5aebc75183874177a79e3bef71be93af 2024-11-14T21:32:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731620000324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:20,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:20,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:20,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/5aebc75183874177a79e3bef71be93af, entries=150, sequenceid=349, filesize=12.0 K 2024-11-14T21:32:20,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/b9a9bbe00a6a42fab966c2f2ec5663e4 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/b9a9bbe00a6a42fab966c2f2ec5663e4 2024-11-14T21:32:20,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731620000331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:20,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:20,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:20,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:20,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/b9a9bbe00a6a42fab966c2f2ec5663e4, entries=150, sequenceid=349, filesize=12.0 K 2024-11-14T21:32:20,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/e7ba4822ee0f498cb12705d247a42175 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/e7ba4822ee0f498cb12705d247a42175 2024-11-14T21:32:20,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731620000354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:20,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/e7ba4822ee0f498cb12705d247a42175, entries=150, sequenceid=349, filesize=12.0 K 2024-11-14T21:32:20,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e9722fc966803fe6d499d80c4851a0fa in 765ms, sequenceid=349, compaction requested=true 2024-11-14T21:32:20,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:20,387 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:20,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:32:20,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:20,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:32:20,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:20,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:32:20,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:20,387 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:20,389 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:20,389 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/B is initiating minor compaction (all files) 2024-11-14T21:32:20,389 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/B in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:20,390 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/766a94b7e3214b60b837f3016946ae0a, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/dab55aefa72149f782a165b82b8b0214, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/b9a9bbe00a6a42fab966c2f2ec5663e4] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=36.8 K 2024-11-14T21:32:20,390 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 766a94b7e3214b60b837f3016946ae0a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731619937025 2024-11-14T21:32:20,391 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting dab55aefa72149f782a165b82b8b0214, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731619938243 2024-11-14T21:32:20,391 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:20,391 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/A is initiating minor compaction (all files) 2024-11-14T21:32:20,391 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/A in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:20,391 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8e2f91dce5784d3c819dc91080a657e2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/e94a81d31ae04fc783a3cb296c36b11f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/5aebc75183874177a79e3bef71be93af] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=39.2 K 2024-11-14T21:32:20,392 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting b9a9bbe00a6a42fab966c2f2ec5663e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731619938941 2024-11-14T21:32:20,393 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e2f91dce5784d3c819dc91080a657e2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731619937025 2024-11-14T21:32:20,394 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting e94a81d31ae04fc783a3cb296c36b11f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731619938229 2024-11-14T21:32:20,395 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5aebc75183874177a79e3bef71be93af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731619938941 2024-11-14T21:32:20,421 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#B#compaction#74 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:20,422 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/4245dd39acfe4f6ea4c1aeeb95375f78 is 50, key is test_row_0/B:col10/1731619938941/Put/seqid=0 2024-11-14T21:32:20,423 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#A#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:20,424 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/12abc3825cfe421c833b05805c53a115 is 50, key is test_row_0/A:col10/1731619938941/Put/seqid=0 2024-11-14T21:32:20,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T21:32:20,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:20,447 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T21:32:20,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:20,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:20,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:20,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:20,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:20,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:20,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/9b0b71dcc7244487bfa3e94d85f44683 is 50, key is test_row_0/A:col10/1731619939666/Put/seqid=0 2024-11-14T21:32:20,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741911_1087 (size=13153) 2024-11-14T21:32:20,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741912_1088 (size=13153) 2024-11-14T21:32:20,527 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/12abc3825cfe421c833b05805c53a115 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/12abc3825cfe421c833b05805c53a115 2024-11-14T21:32:20,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741913_1089 (size=12301) 2024-11-14T21:32:20,531 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/9b0b71dcc7244487bfa3e94d85f44683 2024-11-14T21:32:20,550 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/A of e9722fc966803fe6d499d80c4851a0fa into 12abc3825cfe421c833b05805c53a115(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:20,550 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:20,550 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/A, priority=13, startTime=1731619940386; duration=0sec 2024-11-14T21:32:20,562 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:20,563 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:A 2024-11-14T21:32:20,563 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:20,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/1224a70d81e24ccd9393635c931ab4bb is 50, key is test_row_0/B:col10/1731619939666/Put/seqid=0 2024-11-14T21:32:20,569 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:20,570 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/C is initiating minor compaction (all files) 2024-11-14T21:32:20,570 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/C in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:20,570 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/7111af08da8046bc87246c0a1eaabe6f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/3879c8ca35944a1bbdd2b76756897e69, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/e7ba4822ee0f498cb12705d247a42175] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=36.7 K 2024-11-14T21:32:20,575 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7111af08da8046bc87246c0a1eaabe6f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731619937025 2024-11-14T21:32:20,576 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3879c8ca35944a1bbdd2b76756897e69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731619938243 2024-11-14T21:32:20,576 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7ba4822ee0f498cb12705d247a42175, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731619938941 2024-11-14T21:32:20,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741914_1090 (size=12301) 2024-11-14T21:32:20,587 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/1224a70d81e24ccd9393635c931ab4bb 2024-11-14T21:32:20,591 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#C#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:20,591 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/a5eb15f90d75481ea504d1cfdbf4044c is 50, key is test_row_0/C:col10/1731619938941/Put/seqid=0 2024-11-14T21:32:20,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T21:32:20,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741915_1091 (size=13119) 2024-11-14T21:32:20,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/7c2bc4ae0bf44779aae55bb24addd759 is 50, key is test_row_0/C:col10/1731619939666/Put/seqid=0 2024-11-14T21:32:20,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741916_1092 (size=12301) 2024-11-14T21:32:20,720 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/7c2bc4ae0bf44779aae55bb24addd759 2024-11-14T21:32:20,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/9b0b71dcc7244487bfa3e94d85f44683 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/9b0b71dcc7244487bfa3e94d85f44683 2024-11-14T21:32:20,744 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/9b0b71dcc7244487bfa3e94d85f44683, entries=150, sequenceid=361, filesize=12.0 K 2024-11-14T21:32:20,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/1224a70d81e24ccd9393635c931ab4bb as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/1224a70d81e24ccd9393635c931ab4bb 2024-11-14T21:32:20,783 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/1224a70d81e24ccd9393635c931ab4bb, entries=150, sequenceid=361, filesize=12.0 K 2024-11-14T21:32:20,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/7c2bc4ae0bf44779aae55bb24addd759 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/7c2bc4ae0bf44779aae55bb24addd759 2024-11-14T21:32:20,798 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/7c2bc4ae0bf44779aae55bb24addd759, entries=150, sequenceid=361, filesize=12.0 K 2024-11-14T21:32:20,803 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for e9722fc966803fe6d499d80c4851a0fa in 356ms, sequenceid=361, compaction requested=false 2024-11-14T21:32:20,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:20,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:20,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-14T21:32:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-14T21:32:20,829 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-14T21:32:20,829 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3450 sec 2024-11-14T21:32:20,837 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.3770 sec 2024-11-14T21:32:20,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:20,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T21:32:20,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:20,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:20,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:20,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:20,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:20,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:20,911 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/4245dd39acfe4f6ea4c1aeeb95375f78 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/4245dd39acfe4f6ea4c1aeeb95375f78 2024-11-14T21:32:20,926 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/B of e9722fc966803fe6d499d80c4851a0fa into 4245dd39acfe4f6ea4c1aeeb95375f78(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:20,926 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:20,927 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/B, priority=13, startTime=1731619940387; duration=0sec 2024-11-14T21:32:20,927 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:20,927 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:B 2024-11-14T21:32:20,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/a4a2d279d7f34dc1a0596695ffa67dd9 is 50, key is test_row_0/A:col10/1731619940866/Put/seqid=0 2024-11-14T21:32:20,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741917_1093 (size=14741) 2024-11-14T21:32:20,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/a4a2d279d7f34dc1a0596695ffa67dd9 2024-11-14T21:32:21,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/c2f78c23565244b2b814ba41e82907e5 is 50, key is test_row_0/B:col10/1731619940866/Put/seqid=0 2024-11-14T21:32:21,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731619940949, max timestamp=1731619941001Num uniques 859; 2024-11-14T21:32:21,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T21:32:21,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731619940949, max timestamp=1731619941001Num uniques 859; 2024-11-14T21:32:21,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T21:32:21,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741918_1094 (size=12301) 2024-11-14T21:32:21,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731619940949, max timestamp=1731619941004Num uniques 859; 2024-11-14T21:32:21,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T21:32:21,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/c2f78c23565244b2b814ba41e82907e5 2024-11-14T21:32:21,051 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/a5eb15f90d75481ea504d1cfdbf4044c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/a5eb15f90d75481ea504d1cfdbf4044c 2024-11-14T21:32:21,077 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/C of e9722fc966803fe6d499d80c4851a0fa into a5eb15f90d75481ea504d1cfdbf4044c(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:21,077 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:21,077 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/C, priority=13, startTime=1731619940387; duration=0sec 2024-11-14T21:32:21,078 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:21,078 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:C 2024-11-14T21:32:21,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/0cb957e89ba849fdbc9035e2c3ec160b is 50, key is test_row_0/C:col10/1731619940866/Put/seqid=0 2024-11-14T21:32:21,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741919_1095 (size=12301) 2024-11-14T21:32:21,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731620001107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:21,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/0cb957e89ba849fdbc9035e2c3ec160b 2024-11-14T21:32:21,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731620001113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:21,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:21,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:21,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731620001114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:21,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:21,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/a4a2d279d7f34dc1a0596695ffa67dd9 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/a4a2d279d7f34dc1a0596695ffa67dd9 2024-11-14T21:32:21,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/a4a2d279d7f34dc1a0596695ffa67dd9, entries=200, sequenceid=372, filesize=14.4 K 2024-11-14T21:32:21,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/c2f78c23565244b2b814ba41e82907e5 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/c2f78c23565244b2b814ba41e82907e5 2024-11-14T21:32:21,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/c2f78c23565244b2b814ba41e82907e5, entries=150, sequenceid=372, filesize=12.0 K 2024-11-14T21:32:21,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/0cb957e89ba849fdbc9035e2c3ec160b as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/0cb957e89ba849fdbc9035e2c3ec160b 2024-11-14T21:32:21,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/0cb957e89ba849fdbc9035e2c3ec160b, entries=150, sequenceid=372, filesize=12.0 K 2024-11-14T21:32:21,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=135.63 KB/138880 for e9722fc966803fe6d499d80c4851a0fa in 311ms, sequenceid=372, compaction requested=true 2024-11-14T21:32:21,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:21,198 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:21,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:32:21,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:21,199 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:21,208 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:21,208 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/A is initiating minor compaction (all files) 2024-11-14T21:32:21,208 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/A in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:21,209 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/12abc3825cfe421c833b05805c53a115, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/9b0b71dcc7244487bfa3e94d85f44683, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/a4a2d279d7f34dc1a0596695ffa67dd9] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=39.3 K 2024-11-14T21:32:21,209 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:21,209 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/B is initiating minor compaction (all files) 2024-11-14T21:32:21,209 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/B in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:21,210 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/4245dd39acfe4f6ea4c1aeeb95375f78, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/1224a70d81e24ccd9393635c931ab4bb, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/c2f78c23565244b2b814ba41e82907e5] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=36.9 K 2024-11-14T21:32:21,210 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12abc3825cfe421c833b05805c53a115, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731619938941 2024-11-14T21:32:21,211 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 4245dd39acfe4f6ea4c1aeeb95375f78, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731619938941 2024-11-14T21:32:21,211 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b0b71dcc7244487bfa3e94d85f44683, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731619939662 2024-11-14T21:32:21,211 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 1224a70d81e24ccd9393635c931ab4bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731619939662 2024-11-14T21:32:21,212 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4a2d279d7f34dc1a0596695ffa67dd9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731619940851 2024-11-14T21:32:21,212 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting c2f78c23565244b2b814ba41e82907e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731619940851 2024-11-14T21:32:21,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:32:21,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:21,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:32:21,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:21,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731619941026, max timestamp=1731619941245Num uniques 1009; 2024-11-14T21:32:21,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-14T21:32:21,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731619941026, max timestamp=1731619941245Num uniques 1009; 2024-11-14T21:32:21,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-14T21:32:21,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:21,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=80.51 KB heapSize=223.72 KB 2024-11-14T21:32:21,265 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#B#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:21,265 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#A#compaction#83 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:21,265 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/e4bc93b6e6184bebb1442ee64a24f742 is 50, key is test_row_0/B:col10/1731619940866/Put/seqid=0 2024-11-14T21:32:21,266 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/ab8620c3508440e287ab36aa4c54ca44 is 50, key is test_row_0/A:col10/1731619940866/Put/seqid=0 2024-11-14T21:32:21,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:21,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T21:32:21,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:21,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:32:21,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:21,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:32:21,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741921_1097 (size=13255) 2024-11-14T21:32:21,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741920_1096 (size=13255) 2024-11-14T21:32:21,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/2a7933f94fa34f0389c3a9610e637c28 is 50, key is test_row_0/A:col10/1731619941256/Put/seqid=0 2024-11-14T21:32:21,329 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/ab8620c3508440e287ab36aa4c54ca44 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/ab8620c3508440e287ab36aa4c54ca44 2024-11-14T21:32:21,338 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/A of e9722fc966803fe6d499d80c4851a0fa into ab8620c3508440e287ab36aa4c54ca44(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:21,339 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:21,339 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/A, priority=13, startTime=1731619941198; duration=0sec 2024-11-14T21:32:21,339 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:21,339 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:A 2024-11-14T21:32:21,339 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:21,340 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:21,340 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/C is initiating minor compaction (all files) 2024-11-14T21:32:21,341 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/C in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:21,341 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/a5eb15f90d75481ea504d1cfdbf4044c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/7c2bc4ae0bf44779aae55bb24addd759, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/0cb957e89ba849fdbc9035e2c3ec160b] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=36.8 K 2024-11-14T21:32:21,341 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5eb15f90d75481ea504d1cfdbf4044c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731619938941 2024-11-14T21:32:21,342 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c2bc4ae0bf44779aae55bb24addd759, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731619939662 2024-11-14T21:32:21,343 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cb957e89ba849fdbc9035e2c3ec160b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731619940851 2024-11-14T21:32:21,351 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/e4bc93b6e6184bebb1442ee64a24f742 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/e4bc93b6e6184bebb1442ee64a24f742 2024-11-14T21:32:21,367 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/B of e9722fc966803fe6d499d80c4851a0fa into e4bc93b6e6184bebb1442ee64a24f742(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:21,367 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:21,367 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/B, priority=13, startTime=1731619941199; duration=0sec 2024-11-14T21:32:21,368 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:21,368 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:B 2024-11-14T21:32:21,386 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#C#compaction#86 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:21,386 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/a6564387c00644e2b5e315260953dc79 is 50, key is test_row_0/C:col10/1731619940866/Put/seqid=0 2024-11-14T21:32:21,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741922_1098 (size=12301) 2024-11-14T21:32:21,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731620001380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:21,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:21,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731620001387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:21,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:21,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731620001415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:21,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:21,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741923_1099 (size=13221) 2024-11-14T21:32:21,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731620001521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:21,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:21,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731620001522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:21,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:21,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731620001535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T21:32:21,687 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:32:21,694 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:32:21,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-14T21:32:21,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T21:32:21,704 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:32:21,705 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:32:21,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:32:21,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:21,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731620001731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731620001731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:21,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:21,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:21,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:21,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731620001740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:21,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:21,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:21,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/2a7933f94fa34f0389c3a9610e637c28 2024-11-14T21:32:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T21:32:21,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/52b572e35852449f88a2eb0078bb9597 is 50, key is test_row_0/B:col10/1731619941256/Put/seqid=0 2024-11-14T21:32:21,860 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/a6564387c00644e2b5e315260953dc79 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/a6564387c00644e2b5e315260953dc79 2024-11-14T21:32:21,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T21:32:21,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:21,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:21,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:21,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:21,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:21,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741924_1100 (size=12301) 2024-11-14T21:32:21,875 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/C of e9722fc966803fe6d499d80c4851a0fa into a6564387c00644e2b5e315260953dc79(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:21,875 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:21,875 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/C, priority=13, startTime=1731619941224; duration=0sec 2024-11-14T21:32:21,875 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:21,875 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:C 2024-11-14T21:32:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:21,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/52b572e35852449f88a2eb0078bb9597 2024-11-14T21:32:21,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/e07eec0a43584aacb4a2521290bdfec6 is 50, key is test_row_0/C:col10/1731619941256/Put/seqid=0 2024-11-14T21:32:21,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741925_1101 (size=12301) 2024-11-14T21:32:21,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/e07eec0a43584aacb4a2521290bdfec6 2024-11-14T21:32:21,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/2a7933f94fa34f0389c3a9610e637c28 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/2a7933f94fa34f0389c3a9610e637c28 2024-11-14T21:32:21,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/2a7933f94fa34f0389c3a9610e637c28, entries=150, sequenceid=414, filesize=12.0 K 2024-11-14T21:32:21,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/52b572e35852449f88a2eb0078bb9597 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/52b572e35852449f88a2eb0078bb9597 2024-11-14T21:32:21,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/52b572e35852449f88a2eb0078bb9597, entries=150, sequenceid=414, filesize=12.0 K 2024-11-14T21:32:21,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/e07eec0a43584aacb4a2521290bdfec6 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/e07eec0a43584aacb4a2521290bdfec6 2024-11-14T21:32:21,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/e07eec0a43584aacb4a2521290bdfec6, entries=150, sequenceid=414, filesize=12.0 K 2024-11-14T21:32:21,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~223.67 KB/229040, currentSize=100.63 KB/103050 for e9722fc966803fe6d499d80c4851a0fa in 686ms, sequenceid=414, compaction requested=false 2024-11-14T21:32:21,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T21:32:22,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T21:32:22,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:22,029 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T21:32:22,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:22,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:22,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:22,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:22,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:22,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:22,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/8a190b9f5ef7462fbaa071a2c26962d2 is 50, key is test_row_0/A:col10/1731619941348/Put/seqid=0 2024-11-14T21:32:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:22,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:22,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741926_1102 (size=12301) 2024-11-14T21:32:22,071 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/8a190b9f5ef7462fbaa071a2c26962d2 2024-11-14T21:32:22,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/80239a6bd1ef47b0aa08d2bd26a6b4e7 is 50, key is test_row_0/B:col10/1731619941348/Put/seqid=0 2024-11-14T21:32:22,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731620002082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:22,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731620002085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:22,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731620002090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:22,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:22,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741927_1103 (size=12301) 2024-11-14T21:32:22,146 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/80239a6bd1ef47b0aa08d2bd26a6b4e7 2024-11-14T21:32:22,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/6a7c342e955645a98d7b54eee4e97881 is 50, key is test_row_0/C:col10/1731619941348/Put/seqid=0 2024-11-14T21:32:22,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731620002201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:22,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731620002201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:22,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:22,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731620002201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:22,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:22,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741928_1104 (size=12301) 2024-11-14T21:32:22,224 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/6a7c342e955645a98d7b54eee4e97881 2024-11-14T21:32:22,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/8a190b9f5ef7462fbaa071a2c26962d2 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8a190b9f5ef7462fbaa071a2c26962d2 2024-11-14T21:32:22,265 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8a190b9f5ef7462fbaa071a2c26962d2, entries=150, sequenceid=435, filesize=12.0 K 2024-11-14T21:32:22,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/80239a6bd1ef47b0aa08d2bd26a6b4e7 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/80239a6bd1ef47b0aa08d2bd26a6b4e7 2024-11-14T21:32:22,274 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/80239a6bd1ef47b0aa08d2bd26a6b4e7, entries=150, sequenceid=435, filesize=12.0 K 2024-11-14T21:32:22,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/6a7c342e955645a98d7b54eee4e97881 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/6a7c342e955645a98d7b54eee4e97881 2024-11-14T21:32:22,291 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/6a7c342e955645a98d7b54eee4e97881, entries=150, sequenceid=435, filesize=12.0 K 2024-11-14T21:32:22,292 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e9722fc966803fe6d499d80c4851a0fa in 263ms, sequenceid=435, compaction requested=true 2024-11-14T21:32:22,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:22,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:22,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-14T21:32:22,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-14T21:32:22,297 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-14T21:32:22,297 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 589 msec 2024-11-14T21:32:22,299 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 603 msec 2024-11-14T21:32:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T21:32:22,331 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:32:22,332 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:32:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-14T21:32:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T21:32:22,335 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:32:22,338 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:32:22,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:32:22,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T21:32:22,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:22,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:22,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:22,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:22,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:22,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T21:32:22,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/dadef318806a412f95218f3f5401ba03 is 50, key is test_row_0/A:col10/1731619942088/Put/seqid=0 2024-11-14T21:32:22,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T21:32:22,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:22,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:22,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:22,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:22,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:22,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731620002506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:22,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:22,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741929_1105 (size=14741) 2024-11-14T21:32:22,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731620002508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:22,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:22,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731620002510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:22,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:22,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:22,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731620002613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:22,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:22,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36090 deadline: 1731620002621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:22,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731620002622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:22,630 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_1' in e9722fc966803fe6d499d80c4851a0fa of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18558 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:22,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731620002622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:22,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T21:32:22,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T21:32:22,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:22,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:22,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:22,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:22,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:22,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T21:32:22,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:22,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:22,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:22,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:22,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:22,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731620002820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:22,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731620002842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:22,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731620002841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:22,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:22,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/dadef318806a412f95218f3f5401ba03 2024-11-14T21:32:22,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/9461b20a1e1e41548a2053e250e19bc0 is 50, key is test_row_0/B:col10/1731619942088/Put/seqid=0 2024-11-14T21:32:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T21:32:22,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741930_1106 (size=12301) 2024-11-14T21:32:22,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/9461b20a1e1e41548a2053e250e19bc0 2024-11-14T21:32:22,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T21:32:22,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:22,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:22,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:22,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:22,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:23,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/416725ddfc5b44e2abda50bb49e52988 is 50, key is test_row_0/C:col10/1731619942088/Put/seqid=0 2024-11-14T21:32:23,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741931_1107 (size=12301) 2024-11-14T21:32:23,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T21:32:23,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:23,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:23,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:23,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:23,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:23,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731620003135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:23,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:23,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:23,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:23,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731620003161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:23,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:23,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:23,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:23,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731620003171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:23,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:23,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:23,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T21:32:23,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:23,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:23,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:23,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:23,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:23,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T21:32:23,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:23,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:23,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:23,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:23,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T21:32:23,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/416725ddfc5b44e2abda50bb49e52988 2024-11-14T21:32:23,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/dadef318806a412f95218f3f5401ba03 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/dadef318806a412f95218f3f5401ba03 2024-11-14T21:32:23,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/dadef318806a412f95218f3f5401ba03, entries=200, sequenceid=450, filesize=14.4 K 2024-11-14T21:32:23,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/9461b20a1e1e41548a2053e250e19bc0 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/9461b20a1e1e41548a2053e250e19bc0 2024-11-14T21:32:23,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/9461b20a1e1e41548a2053e250e19bc0, entries=150, sequenceid=450, filesize=12.0 K 2024-11-14T21:32:23,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/416725ddfc5b44e2abda50bb49e52988 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/416725ddfc5b44e2abda50bb49e52988 2024-11-14T21:32:23,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/416725ddfc5b44e2abda50bb49e52988, entries=150, sequenceid=450, filesize=12.0 K 2024-11-14T21:32:23,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for e9722fc966803fe6d499d80c4851a0fa in 1134ms, sequenceid=450, compaction requested=true 2024-11-14T21:32:23,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:23,558 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:32:23,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:32:23,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:23,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:32:23,561 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52598 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:32:23,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:23,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:32:23,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:32:23,561 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/A is initiating minor compaction (all files) 2024-11-14T21:32:23,561 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/A in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:23,561 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/ab8620c3508440e287ab36aa4c54ca44, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/2a7933f94fa34f0389c3a9610e637c28, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8a190b9f5ef7462fbaa071a2c26962d2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/dadef318806a412f95218f3f5401ba03] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=51.4 K 2024-11-14T21:32:23,562 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting ab8620c3508440e287ab36aa4c54ca44, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731619940851 2024-11-14T21:32:23,562 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:32:23,567 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a7933f94fa34f0389c3a9610e637c28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731619941026 2024-11-14T21:32:23,577 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a190b9f5ef7462fbaa071a2c26962d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731619941348 2024-11-14T21:32:23,577 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting dadef318806a412f95218f3f5401ba03, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731619942064 2024-11-14T21:32:23,577 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:32:23,577 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/B is initiating minor compaction (all files) 2024-11-14T21:32:23,578 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/B in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:23,578 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/e4bc93b6e6184bebb1442ee64a24f742, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/52b572e35852449f88a2eb0078bb9597, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/80239a6bd1ef47b0aa08d2bd26a6b4e7, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/9461b20a1e1e41548a2053e250e19bc0] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=49.0 K 2024-11-14T21:32:23,583 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4bc93b6e6184bebb1442ee64a24f742, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731619940851 2024-11-14T21:32:23,588 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52b572e35852449f88a2eb0078bb9597, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731619941026 2024-11-14T21:32:23,592 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80239a6bd1ef47b0aa08d2bd26a6b4e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731619941348 2024-11-14T21:32:23,598 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9461b20a1e1e41548a2053e250e19bc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731619942064 2024-11-14T21:32:23,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T21:32:23,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:23,606 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T21:32:23,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:23,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:23,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:23,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:23,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:23,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:23,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/f53db4e96f0f44419999ca46c19d5b91 is 50, key is test_row_0/A:col10/1731619942504/Put/seqid=0 2024-11-14T21:32:23,638 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#A#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:23,639 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/4dec0a85f0ae43bd93b4796927d71802 is 50, key is test_row_0/A:col10/1731619942088/Put/seqid=0 2024-11-14T21:32:23,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:23,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. as already flushing 2024-11-14T21:32:23,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741933_1109 (size=13391) 2024-11-14T21:32:23,705 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#B#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:23,706 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/72d381a861fc46b8a062957c17a855d8 is 50, key is test_row_0/B:col10/1731619942088/Put/seqid=0 2024-11-14T21:32:23,708 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/4dec0a85f0ae43bd93b4796927d71802 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/4dec0a85f0ae43bd93b4796927d71802 2024-11-14T21:32:23,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741932_1108 (size=12301) 2024-11-14T21:32:23,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741934_1110 (size=13391) 2024-11-14T21:32:23,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731620003734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:23,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:23,751 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-14T21:32:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731620003735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:23,751 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:32:23,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731620003744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:23,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:23,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:23,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:23,758 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:23,758 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/A of e9722fc966803fe6d499d80c4851a0fa into 4dec0a85f0ae43bd93b4796927d71802(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:23,759 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:23,759 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/A, priority=12, startTime=1731619943558; duration=0sec 2024-11-14T21:32:23,759 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:23,759 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:A 2024-11-14T21:32:23,759 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:32:23,759 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-14T21:32:23,759 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:32:23,759 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:23,760 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-14T21:32:23,760 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:32:23,760 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:23,760 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:23,760 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:23,760 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:23,760 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:32:23,761 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:32:23,763 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:32:23,763 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/C is initiating minor compaction (all files) 2024-11-14T21:32:23,763 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/C in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:23,763 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/a6564387c00644e2b5e315260953dc79, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/e07eec0a43584aacb4a2521290bdfec6, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/6a7c342e955645a98d7b54eee4e97881, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/416725ddfc5b44e2abda50bb49e52988] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=48.9 K 2024-11-14T21:32:23,764 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting a6564387c00644e2b5e315260953dc79, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731619940851 2024-11-14T21:32:23,764 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting e07eec0a43584aacb4a2521290bdfec6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731619941026 2024-11-14T21:32:23,765 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a7c342e955645a98d7b54eee4e97881, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731619941348 2024-11-14T21:32:23,765 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 416725ddfc5b44e2abda50bb49e52988, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731619942064 2024-11-14T21:32:23,766 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:32:23,767 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-14T21:32:23,767 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:32:23,767 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:23,767 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:23,767 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:32:23,784 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#C#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:23,785 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/107352098a564fb68c4c98275d542bbb is 50, key is test_row_0/C:col10/1731619942088/Put/seqid=0 2024-11-14T21:32:23,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741935_1111 (size=13357) 2024-11-14T21:32:23,800 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/107352098a564fb68c4c98275d542bbb as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/107352098a564fb68c4c98275d542bbb 2024-11-14T21:32:23,807 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/C of e9722fc966803fe6d499d80c4851a0fa into 107352098a564fb68c4c98275d542bbb(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:23,807 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:23,807 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/C, priority=12, startTime=1731619943561; duration=0sec 2024-11-14T21:32:23,808 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:23,808 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:C 2024-11-14T21:32:23,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:23,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731620003861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731620003861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:23,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:23,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:23,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:23,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:23,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:23,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:23,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731620003861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:23,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731620003920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:23,921 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_1' in e9722fc966803fe6d499d80c4851a0fa of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18315 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:23,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:23,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:23,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:24,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:24,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731620004071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731620004071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:24,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:24,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:24,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:24,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:24,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:24,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:24,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731620004081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:24,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:24,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:24,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:24,112 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/f53db4e96f0f44419999ca46c19d5b91 2024-11-14T21:32:24,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/9f7884628dd6409ea58887bbf75bf0f8 is 50, key is test_row_0/B:col10/1731619942504/Put/seqid=0 2024-11-14T21:32:24,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741936_1112 (size=12301) 2024-11-14T21:32:24,152 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/9f7884628dd6409ea58887bbf75bf0f8 2024-11-14T21:32:24,158 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/72d381a861fc46b8a062957c17a855d8 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/72d381a861fc46b8a062957c17a855d8 2024-11-14T21:32:24,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/713fdf581625425d9a6c7de799773fd6 is 50, key is test_row_0/C:col10/1731619942504/Put/seqid=0 2024-11-14T21:32:24,167 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/B of e9722fc966803fe6d499d80c4851a0fa into 72d381a861fc46b8a062957c17a855d8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:24,167 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:24,167 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/B, priority=12, startTime=1731619943561; duration=0sec 2024-11-14T21:32:24,167 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:24,167 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:B 2024-11-14T21:32:24,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741937_1113 (size=12301) 2024-11-14T21:32:24,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:24,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731620004380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:24,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:24,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:24,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:24,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:24,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36098 deadline: 1731620004380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:24,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:24,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:24,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:24,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:24,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36070 deadline: 1731620004391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:32:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T21:32:24,569 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/713fdf581625425d9a6c7de799773fd6 2024-11-14T21:32:24,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/f53db4e96f0f44419999ca46c19d5b91 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/f53db4e96f0f44419999ca46c19d5b91 2024-11-14T21:32:24,579 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/f53db4e96f0f44419999ca46c19d5b91, entries=150, sequenceid=468, filesize=12.0 K 2024-11-14T21:32:24,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/9f7884628dd6409ea58887bbf75bf0f8 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/9f7884628dd6409ea58887bbf75bf0f8 2024-11-14T21:32:24,583 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/9f7884628dd6409ea58887bbf75bf0f8, entries=150, sequenceid=468, filesize=12.0 K 2024-11-14T21:32:24,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/713fdf581625425d9a6c7de799773fd6 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/713fdf581625425d9a6c7de799773fd6 2024-11-14T21:32:24,588 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/713fdf581625425d9a6c7de799773fd6, entries=150, sequenceid=468, filesize=12.0 K 2024-11-14T21:32:24,589 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e9722fc966803fe6d499d80c4851a0fa in 983ms, sequenceid=468, compaction requested=false 2024-11-14T21:32:24,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:24,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:24,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-14T21:32:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-14T21:32:24,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-14T21:32:24,593 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2530 sec 2024-11-14T21:32:24,594 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.2610 sec 2024-11-14T21:32:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:24,898 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-14T21:32:24,898 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:32:24,898 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:24,898 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:24,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:32:24,899 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-14T21:32:24,899 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:32:24,899 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:24,899 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:24,899 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:32:24,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T21:32:24,903 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-14T21:32:24,903 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:32:24,903 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:24,904 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:24,904 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:32:24,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:24,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:24,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:24,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:24,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:24,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:24,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/8d3d5759d1c2442b862dd5489d5b5136 is 50, key is test_row_0/A:col10/1731619943721/Put/seqid=0 2024-11-14T21:32:24,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741938_1114 (size=12301) 2024-11-14T21:32:25,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/8d3d5759d1c2442b862dd5489d5b5136 2024-11-14T21:32:25,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/9a25256686414ddcafc7574f2a4fffa3 is 50, key is test_row_0/B:col10/1731619943721/Put/seqid=0 2024-11-14T21:32:25,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741939_1115 (size=12301) 2024-11-14T21:32:25,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/9a25256686414ddcafc7574f2a4fffa3 2024-11-14T21:32:25,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/fa1136f073824519a41d7e6935af4819 is 50, key is test_row_0/C:col10/1731619943721/Put/seqid=0 2024-11-14T21:32:25,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741940_1116 (size=12301) 2024-11-14T21:32:26,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/fa1136f073824519a41d7e6935af4819 2024-11-14T21:32:26,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/8d3d5759d1c2442b862dd5489d5b5136 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8d3d5759d1c2442b862dd5489d5b5136 2024-11-14T21:32:26,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8d3d5759d1c2442b862dd5489d5b5136, entries=150, sequenceid=488, filesize=12.0 K 2024-11-14T21:32:26,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/9a25256686414ddcafc7574f2a4fffa3 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/9a25256686414ddcafc7574f2a4fffa3 2024-11-14T21:32:26,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/9a25256686414ddcafc7574f2a4fffa3, entries=150, sequenceid=488, filesize=12.0 K 2024-11-14T21:32:26,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/fa1136f073824519a41d7e6935af4819 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/fa1136f073824519a41d7e6935af4819 2024-11-14T21:32:26,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/fa1136f073824519a41d7e6935af4819, entries=150, sequenceid=488, filesize=12.0 K 2024-11-14T21:32:26,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for e9722fc966803fe6d499d80c4851a0fa in 1332ms, sequenceid=488, compaction requested=true 2024-11-14T21:32:26,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:26,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:32:26,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:26,235 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:26,235 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:26,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:32:26,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:26,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e9722fc966803fe6d499d80c4851a0fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:32:26,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:26,237 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:26,237 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/A is initiating minor compaction (all files) 2024-11-14T21:32:26,237 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/A in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:26,237 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/4dec0a85f0ae43bd93b4796927d71802, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/f53db4e96f0f44419999ca46c19d5b91, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8d3d5759d1c2442b862dd5489d5b5136] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=37.1 K 2024-11-14T21:32:26,238 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:26,238 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/B is initiating minor compaction (all files) 2024-11-14T21:32:26,238 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dec0a85f0ae43bd93b4796927d71802, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731619942064 2024-11-14T21:32:26,238 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/B in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:26,238 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/72d381a861fc46b8a062957c17a855d8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/9f7884628dd6409ea58887bbf75bf0f8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/9a25256686414ddcafc7574f2a4fffa3] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=37.1 K 2024-11-14T21:32:26,238 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting f53db4e96f0f44419999ca46c19d5b91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731619942496 2024-11-14T21:32:26,238 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 72d381a861fc46b8a062957c17a855d8, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731619942064 2024-11-14T21:32:26,239 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d3d5759d1c2442b862dd5489d5b5136, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731619943711 2024-11-14T21:32:26,239 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f7884628dd6409ea58887bbf75bf0f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731619942496 2024-11-14T21:32:26,240 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a25256686414ddcafc7574f2a4fffa3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731619943711 2024-11-14T21:32:26,256 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#B#compaction#104 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:26,257 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#A#compaction#105 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:26,257 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/71bfc07cc4e144028f3b8fd1c98d11b7 is 50, key is test_row_0/A:col10/1731619943721/Put/seqid=0 2024-11-14T21:32:26,257 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/cec05a9782fc4f749d5cc9d362428b49 is 50, key is test_row_0/B:col10/1731619943721/Put/seqid=0 2024-11-14T21:32:26,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741941_1117 (size=13493) 2024-11-14T21:32:26,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741942_1118 (size=13493) 2024-11-14T21:32:26,269 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/71bfc07cc4e144028f3b8fd1c98d11b7 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/71bfc07cc4e144028f3b8fd1c98d11b7 2024-11-14T21:32:26,274 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/A of e9722fc966803fe6d499d80c4851a0fa into 71bfc07cc4e144028f3b8fd1c98d11b7(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:26,274 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:26,275 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/A, priority=13, startTime=1731619946234; duration=0sec 2024-11-14T21:32:26,275 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:26,275 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:A 2024-11-14T21:32:26,275 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:26,276 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:26,276 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): e9722fc966803fe6d499d80c4851a0fa/C is initiating minor compaction (all files) 2024-11-14T21:32:26,276 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e9722fc966803fe6d499d80c4851a0fa/C in TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:26,276 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/107352098a564fb68c4c98275d542bbb, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/713fdf581625425d9a6c7de799773fd6, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/fa1136f073824519a41d7e6935af4819] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp, totalSize=37.1 K 2024-11-14T21:32:26,276 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 107352098a564fb68c4c98275d542bbb, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731619942064 2024-11-14T21:32:26,277 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 713fdf581625425d9a6c7de799773fd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731619942496 2024-11-14T21:32:26,277 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa1136f073824519a41d7e6935af4819, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731619943711 2024-11-14T21:32:26,286 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e9722fc966803fe6d499d80c4851a0fa#C#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:26,286 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/e6f5ab5177f64c95a9f7a1e65c648ca1 is 50, key is test_row_0/C:col10/1731619943721/Put/seqid=0 2024-11-14T21:32:26,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741943_1119 (size=13459) 2024-11-14T21:32:26,297 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/e6f5ab5177f64c95a9f7a1e65c648ca1 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/e6f5ab5177f64c95a9f7a1e65c648ca1 2024-11-14T21:32:26,305 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/C of e9722fc966803fe6d499d80c4851a0fa into e6f5ab5177f64c95a9f7a1e65c648ca1(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:26,305 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:26,305 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/C, priority=13, startTime=1731619946235; duration=0sec 2024-11-14T21:32:26,306 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:26,306 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:C 2024-11-14T21:32:26,374 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T21:32:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T21:32:26,512 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:32:26,670 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/cec05a9782fc4f749d5cc9d362428b49 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/cec05a9782fc4f749d5cc9d362428b49 2024-11-14T21:32:26,681 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e9722fc966803fe6d499d80c4851a0fa/B of e9722fc966803fe6d499d80c4851a0fa into cec05a9782fc4f749d5cc9d362428b49(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:26,681 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e9722fc966803fe6d499d80c4851a0fa: 2024-11-14T21:32:26,681 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., storeName=e9722fc966803fe6d499d80c4851a0fa/B, priority=13, startTime=1731619946235; duration=0sec 2024-11-14T21:32:26,681 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:26,681 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e9722fc966803fe6d499d80c4851a0fa:B 2024-11-14T21:32:32,644 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-14T21:32:32,644 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:32:32,644 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:32,645 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:32,645 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:32:34,013 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-14T21:32:34,013 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:32:34,013 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:34,013 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:34,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T21:32:34,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-14T21:32:34,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 121 2024-11-14T21:32:34,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 6 2024-11-14T21:32:34,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-11-14T21:32:34,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 122 2024-11-14T21:32:34,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T21:32:34,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1598 2024-11-14T21:32:34,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1586 2024-11-14T21:32:34,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T21:32:34,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 770 2024-11-14T21:32:34,014 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:32:34,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2301 rows 2024-11-14T21:32:34,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 770 2024-11-14T21:32:34,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2295 rows 2024-11-14T21:32:34,015 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T21:32:34,015 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T21:32:34,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:34,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:34,015 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:32:34,020 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T21:32:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T21:32:34,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T21:32:34,044 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731619954043"}]},"ts":"1731619954043"} 2024-11-14T21:32:34,047 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T21:32:34,047 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T21:32:34,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T21:32:34,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e9722fc966803fe6d499d80c4851a0fa, UNASSIGN}] 2024-11-14T21:32:34,054 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e9722fc966803fe6d499d80c4851a0fa, UNASSIGN 2024-11-14T21:32:34,056 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=e9722fc966803fe6d499d80c4851a0fa, regionState=CLOSING, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:34,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e9722fc966803fe6d499d80c4851a0fa, UNASSIGN because future has completed 2024-11-14T21:32:34,063 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T21:32:34,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560}] 2024-11-14T21:32:34,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T21:32:34,221 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(122): Close e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:34,221 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T21:32:34,221 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1722): Closing e9722fc966803fe6d499d80c4851a0fa, disabling compactions & flushes 2024-11-14T21:32:34,221 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:34,222 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:34,222 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. after waiting 0 ms 2024-11-14T21:32:34,222 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:34,222 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(2902): Flushing e9722fc966803fe6d499d80c4851a0fa 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-14T21:32:34,222 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=A 2024-11-14T21:32:34,222 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:34,222 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=B 2024-11-14T21:32:34,222 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:34,222 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e9722fc966803fe6d499d80c4851a0fa, store=C 2024-11-14T21:32:34,222 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:34,226 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/b7dde761177f429589b96da62cb34da4 is 50, key is test_row_1/A:col10/1731619954011/Put/seqid=0 2024-11-14T21:32:34,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741944_1120 (size=7415) 2024-11-14T21:32:34,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T21:32:34,631 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/b7dde761177f429589b96da62cb34da4 2024-11-14T21:32:34,642 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/e43038eb68974e848e49a0d0e8098455 is 50, key is test_row_1/B:col10/1731619954011/Put/seqid=0 2024-11-14T21:32:34,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741945_1121 (size=7415) 2024-11-14T21:32:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T21:32:35,050 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/e43038eb68974e848e49a0d0e8098455 2024-11-14T21:32:35,062 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/7aca8eff150c495fb5caecfbbf6afe3d is 50, key is test_row_1/C:col10/1731619954011/Put/seqid=0 2024-11-14T21:32:35,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741946_1122 (size=7415) 2024-11-14T21:32:35,078 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/7aca8eff150c495fb5caecfbbf6afe3d 2024-11-14T21:32:35,084 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/A/b7dde761177f429589b96da62cb34da4 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/b7dde761177f429589b96da62cb34da4 2024-11-14T21:32:35,089 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/b7dde761177f429589b96da62cb34da4, entries=50, sequenceid=496, filesize=7.2 K 2024-11-14T21:32:35,090 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/B/e43038eb68974e848e49a0d0e8098455 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/e43038eb68974e848e49a0d0e8098455 2024-11-14T21:32:35,095 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/e43038eb68974e848e49a0d0e8098455, entries=50, sequenceid=496, filesize=7.2 K 2024-11-14T21:32:35,096 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/.tmp/C/7aca8eff150c495fb5caecfbbf6afe3d as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/7aca8eff150c495fb5caecfbbf6afe3d 2024-11-14T21:32:35,100 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/7aca8eff150c495fb5caecfbbf6afe3d, entries=50, sequenceid=496, filesize=7.2 K 2024-11-14T21:32:35,101 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for e9722fc966803fe6d499d80c4851a0fa in 879ms, sequenceid=496, compaction requested=false 2024-11-14T21:32:35,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/d6928d43277c449685ed9f7047e9407a, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/080b148d582d403595569de44bb97cf0, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/27f9dad005da4a708d91c9ff38499ca5, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8ab0e32a6b4c4612b2b616bb76e6cf4b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/3192ff621f264eb7817264e7fa155244, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/b9980a2e6a4e4ac5b7b6c6553e035ca5, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/95dc5bdd525a45e882160dcd5d729c20, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/a3722f8e472d4852aaf285e69a7f4ebd, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/5c98e151b921438dbfca863b565071aa, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/cc423868c51b41f8b91046a200d838e5, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/d59f5757bb60456ebd63d453d3bf8b7c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/f619b3dad91b47168aa808a2c7a84b80, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/c8e62df7dc924f389691e88bb33d440b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/3dc98141148046aaa83d775095c3f393, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/d531922ef80840f1985f4c44f776ac21, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/c4fef458338f42e0b9f3ea81790edf01, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/086e25e1b64a457993893e6eb253d543, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8c76b230914d46a7868f0b92c2c72019, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/a8e27ee8d61746148fef3f6ed94fad93, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/5c74734d7023439ea9f137d0b01d8057, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/6ea605de5b5d41d29ddf5b31b361f335, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/240d2bc2589e4561a19edd8295607633, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8e2f91dce5784d3c819dc91080a657e2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/e94a81d31ae04fc783a3cb296c36b11f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/12abc3825cfe421c833b05805c53a115, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/5aebc75183874177a79e3bef71be93af, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/9b0b71dcc7244487bfa3e94d85f44683, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/a4a2d279d7f34dc1a0596695ffa67dd9, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/ab8620c3508440e287ab36aa4c54ca44, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/2a7933f94fa34f0389c3a9610e637c28, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8a190b9f5ef7462fbaa071a2c26962d2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/dadef318806a412f95218f3f5401ba03, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/4dec0a85f0ae43bd93b4796927d71802, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/f53db4e96f0f44419999ca46c19d5b91, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8d3d5759d1c2442b862dd5489d5b5136] to archive 2024-11-14T21:32:35,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:32:35,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/d6928d43277c449685ed9f7047e9407a to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/d6928d43277c449685ed9f7047e9407a 2024-11-14T21:32:35,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/080b148d582d403595569de44bb97cf0 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/080b148d582d403595569de44bb97cf0 2024-11-14T21:32:35,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/27f9dad005da4a708d91c9ff38499ca5 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/27f9dad005da4a708d91c9ff38499ca5 2024-11-14T21:32:35,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8ab0e32a6b4c4612b2b616bb76e6cf4b to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8ab0e32a6b4c4612b2b616bb76e6cf4b 2024-11-14T21:32:35,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/3192ff621f264eb7817264e7fa155244 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/3192ff621f264eb7817264e7fa155244 2024-11-14T21:32:35,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/b9980a2e6a4e4ac5b7b6c6553e035ca5 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/b9980a2e6a4e4ac5b7b6c6553e035ca5 2024-11-14T21:32:35,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/95dc5bdd525a45e882160dcd5d729c20 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/95dc5bdd525a45e882160dcd5d729c20 2024-11-14T21:32:35,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/a3722f8e472d4852aaf285e69a7f4ebd to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/a3722f8e472d4852aaf285e69a7f4ebd 2024-11-14T21:32:35,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/5c98e151b921438dbfca863b565071aa to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/5c98e151b921438dbfca863b565071aa 2024-11-14T21:32:35,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/cc423868c51b41f8b91046a200d838e5 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/cc423868c51b41f8b91046a200d838e5 2024-11-14T21:32:35,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/d59f5757bb60456ebd63d453d3bf8b7c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/d59f5757bb60456ebd63d453d3bf8b7c 2024-11-14T21:32:35,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/f619b3dad91b47168aa808a2c7a84b80 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/f619b3dad91b47168aa808a2c7a84b80 2024-11-14T21:32:35,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/c8e62df7dc924f389691e88bb33d440b to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/c8e62df7dc924f389691e88bb33d440b 2024-11-14T21:32:35,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/3dc98141148046aaa83d775095c3f393 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/3dc98141148046aaa83d775095c3f393 2024-11-14T21:32:35,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/d531922ef80840f1985f4c44f776ac21 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/d531922ef80840f1985f4c44f776ac21 2024-11-14T21:32:35,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/c4fef458338f42e0b9f3ea81790edf01 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/c4fef458338f42e0b9f3ea81790edf01 2024-11-14T21:32:35,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/086e25e1b64a457993893e6eb253d543 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/086e25e1b64a457993893e6eb253d543 2024-11-14T21:32:35,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8c76b230914d46a7868f0b92c2c72019 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8c76b230914d46a7868f0b92c2c72019 2024-11-14T21:32:35,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/a8e27ee8d61746148fef3f6ed94fad93 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/a8e27ee8d61746148fef3f6ed94fad93 2024-11-14T21:32:35,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/5c74734d7023439ea9f137d0b01d8057 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/5c74734d7023439ea9f137d0b01d8057 2024-11-14T21:32:35,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/6ea605de5b5d41d29ddf5b31b361f335 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/6ea605de5b5d41d29ddf5b31b361f335 2024-11-14T21:32:35,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/240d2bc2589e4561a19edd8295607633 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/240d2bc2589e4561a19edd8295607633 2024-11-14T21:32:35,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8e2f91dce5784d3c819dc91080a657e2 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8e2f91dce5784d3c819dc91080a657e2 2024-11-14T21:32:35,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/e94a81d31ae04fc783a3cb296c36b11f to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/e94a81d31ae04fc783a3cb296c36b11f 2024-11-14T21:32:35,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/12abc3825cfe421c833b05805c53a115 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/12abc3825cfe421c833b05805c53a115 2024-11-14T21:32:35,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/5aebc75183874177a79e3bef71be93af to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/5aebc75183874177a79e3bef71be93af 2024-11-14T21:32:35,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/9b0b71dcc7244487bfa3e94d85f44683 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/9b0b71dcc7244487bfa3e94d85f44683 2024-11-14T21:32:35,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/a4a2d279d7f34dc1a0596695ffa67dd9 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/a4a2d279d7f34dc1a0596695ffa67dd9 2024-11-14T21:32:35,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/ab8620c3508440e287ab36aa4c54ca44 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/ab8620c3508440e287ab36aa4c54ca44 2024-11-14T21:32:35,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/2a7933f94fa34f0389c3a9610e637c28 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/2a7933f94fa34f0389c3a9610e637c28 2024-11-14T21:32:35,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8a190b9f5ef7462fbaa071a2c26962d2 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8a190b9f5ef7462fbaa071a2c26962d2 2024-11-14T21:32:35,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/dadef318806a412f95218f3f5401ba03 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/dadef318806a412f95218f3f5401ba03 2024-11-14T21:32:35,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/4dec0a85f0ae43bd93b4796927d71802 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/4dec0a85f0ae43bd93b4796927d71802 2024-11-14T21:32:35,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/f53db4e96f0f44419999ca46c19d5b91 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/f53db4e96f0f44419999ca46c19d5b91 2024-11-14T21:32:35,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8d3d5759d1c2442b862dd5489d5b5136 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/8d3d5759d1c2442b862dd5489d5b5136 2024-11-14T21:32:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T21:32:35,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/d3b04dc2121d416db88778eee6313f64, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/e5fbe764dd9d48afa2f70aa2b3fcc430, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/f145397daa3a42998267b7abed02bada, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/68a226d5f66f47bc932bde6f63fd649d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/2a48496ac93449f486d74b1e821e7856, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/7f59b63c26be4c2f9d8c1fc177ffdf9f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/47d6a304aa71460e8ec3b89ca4183880, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/ad0c4b5495a74321935adc8db2065c02, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/22c2a504272e49be81276fa62221d2a5, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/47b42f17e242489c830057e493fe0f6d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/7fc04c74fafa49269fa84da6f5909fd1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/ce852fbe903944ddb265b6ed4aad98f0, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/cb2b864e714c4de4a569d54e3d5f8235, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/b24cc1a449d84a0f9ddcf7291dd8303c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/ed84ac0ceb4845f7ab22e4a3980f1f2e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/f70acac4a2c64ace9557c96fa8986b44, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/a3efb3ec66834a78a8c8fbd5ed563913, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/2e4401742c814cdcac36c0fefac9618f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/a90e5794810e45ef8cfa9d4a49ddeb5b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/34621090067e4982bdc537c8b435d495, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/5d1426e47fd9423c8268d70b3af264ee, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/766a94b7e3214b60b837f3016946ae0a, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/bcce44460d2c47b2b583b0d1ea58f12f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/dab55aefa72149f782a165b82b8b0214, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/4245dd39acfe4f6ea4c1aeeb95375f78, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/b9a9bbe00a6a42fab966c2f2ec5663e4, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/1224a70d81e24ccd9393635c931ab4bb, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/e4bc93b6e6184bebb1442ee64a24f742, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/c2f78c23565244b2b814ba41e82907e5, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/52b572e35852449f88a2eb0078bb9597, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/80239a6bd1ef47b0aa08d2bd26a6b4e7, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/72d381a861fc46b8a062957c17a855d8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/9461b20a1e1e41548a2053e250e19bc0, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/9f7884628dd6409ea58887bbf75bf0f8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/9a25256686414ddcafc7574f2a4fffa3] to archive 2024-11-14T21:32:35,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:32:35,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/d3b04dc2121d416db88778eee6313f64 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/d3b04dc2121d416db88778eee6313f64 2024-11-14T21:32:35,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/e5fbe764dd9d48afa2f70aa2b3fcc430 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/e5fbe764dd9d48afa2f70aa2b3fcc430 2024-11-14T21:32:35,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/f145397daa3a42998267b7abed02bada to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/f145397daa3a42998267b7abed02bada 2024-11-14T21:32:35,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/68a226d5f66f47bc932bde6f63fd649d to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/68a226d5f66f47bc932bde6f63fd649d 2024-11-14T21:32:35,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/2a48496ac93449f486d74b1e821e7856 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/2a48496ac93449f486d74b1e821e7856 2024-11-14T21:32:35,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/7f59b63c26be4c2f9d8c1fc177ffdf9f to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/7f59b63c26be4c2f9d8c1fc177ffdf9f 2024-11-14T21:32:35,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/47d6a304aa71460e8ec3b89ca4183880 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/47d6a304aa71460e8ec3b89ca4183880 2024-11-14T21:32:35,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/ad0c4b5495a74321935adc8db2065c02 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/ad0c4b5495a74321935adc8db2065c02 2024-11-14T21:32:35,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/22c2a504272e49be81276fa62221d2a5 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/22c2a504272e49be81276fa62221d2a5 2024-11-14T21:32:35,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/47b42f17e242489c830057e493fe0f6d to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/47b42f17e242489c830057e493fe0f6d 2024-11-14T21:32:35,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/7fc04c74fafa49269fa84da6f5909fd1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/7fc04c74fafa49269fa84da6f5909fd1 2024-11-14T21:32:35,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/ce852fbe903944ddb265b6ed4aad98f0 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/ce852fbe903944ddb265b6ed4aad98f0 2024-11-14T21:32:35,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/cb2b864e714c4de4a569d54e3d5f8235 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/cb2b864e714c4de4a569d54e3d5f8235 2024-11-14T21:32:35,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/b24cc1a449d84a0f9ddcf7291dd8303c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/b24cc1a449d84a0f9ddcf7291dd8303c 2024-11-14T21:32:35,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/ed84ac0ceb4845f7ab22e4a3980f1f2e to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/ed84ac0ceb4845f7ab22e4a3980f1f2e 2024-11-14T21:32:35,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/f70acac4a2c64ace9557c96fa8986b44 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/f70acac4a2c64ace9557c96fa8986b44 2024-11-14T21:32:35,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/a3efb3ec66834a78a8c8fbd5ed563913 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/a3efb3ec66834a78a8c8fbd5ed563913 2024-11-14T21:32:35,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/2e4401742c814cdcac36c0fefac9618f to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/2e4401742c814cdcac36c0fefac9618f 2024-11-14T21:32:35,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/a90e5794810e45ef8cfa9d4a49ddeb5b to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/a90e5794810e45ef8cfa9d4a49ddeb5b 2024-11-14T21:32:35,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/34621090067e4982bdc537c8b435d495 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/34621090067e4982bdc537c8b435d495 2024-11-14T21:32:35,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/5d1426e47fd9423c8268d70b3af264ee to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/5d1426e47fd9423c8268d70b3af264ee 2024-11-14T21:32:35,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/766a94b7e3214b60b837f3016946ae0a to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/766a94b7e3214b60b837f3016946ae0a 2024-11-14T21:32:35,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/bcce44460d2c47b2b583b0d1ea58f12f to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/bcce44460d2c47b2b583b0d1ea58f12f 2024-11-14T21:32:35,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/dab55aefa72149f782a165b82b8b0214 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/dab55aefa72149f782a165b82b8b0214 2024-11-14T21:32:35,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/4245dd39acfe4f6ea4c1aeeb95375f78 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/4245dd39acfe4f6ea4c1aeeb95375f78 2024-11-14T21:32:35,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/b9a9bbe00a6a42fab966c2f2ec5663e4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/b9a9bbe00a6a42fab966c2f2ec5663e4 2024-11-14T21:32:35,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/1224a70d81e24ccd9393635c931ab4bb to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/1224a70d81e24ccd9393635c931ab4bb 2024-11-14T21:32:35,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/e4bc93b6e6184bebb1442ee64a24f742 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/e4bc93b6e6184bebb1442ee64a24f742 2024-11-14T21:32:35,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/c2f78c23565244b2b814ba41e82907e5 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/c2f78c23565244b2b814ba41e82907e5 2024-11-14T21:32:35,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/52b572e35852449f88a2eb0078bb9597 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/52b572e35852449f88a2eb0078bb9597 2024-11-14T21:32:35,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/80239a6bd1ef47b0aa08d2bd26a6b4e7 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/80239a6bd1ef47b0aa08d2bd26a6b4e7 2024-11-14T21:32:35,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/72d381a861fc46b8a062957c17a855d8 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/72d381a861fc46b8a062957c17a855d8 2024-11-14T21:32:35,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/9461b20a1e1e41548a2053e250e19bc0 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/9461b20a1e1e41548a2053e250e19bc0 2024-11-14T21:32:35,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/9f7884628dd6409ea58887bbf75bf0f8 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/9f7884628dd6409ea58887bbf75bf0f8 2024-11-14T21:32:35,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/9a25256686414ddcafc7574f2a4fffa3 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/9a25256686414ddcafc7574f2a4fffa3 2024-11-14T21:32:35,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/6e6807177850408e8bc1d6c2ff772f32, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/d120c9badf9646b2bf930c103a3dd782, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/59910bfa2439438e8405bd12eafa67ec, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/0e1e307d55b74ddd9480787928d976ae, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/5d5a966caf974a9587ef02aeadb09754, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/2cf98273e79148178341c8358c1a249e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/8523ae2d2b9f43eebf5ae51c84188a53, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/8c1d84db42584b8bb000633423971555, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/64a49ec24a83477986c7e6639b027f3d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/bc79a8d2b436441ab7c6739f01c797ea, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/a92f10e1b3bb4a309f8759047b0f8e1b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/5a201678e37a4eaa89a301239bfcd989, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/efe9a8669f624b11a27b5c47fe8f4fe3, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/6b8f1ee2e5c14e04948e524c124200b2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/9b5b4c2a3d2f4ac6ae21dee55f78ecb1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/1833257b4e4249d8a061651e7874f9a9, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/50638068572d44f9a3192d230539c628, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/2270d20a38394b4f90dbcde54c0399ac, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/a79c794842404f30812323be2bf22616, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/5a0e1eabe218454581b87afba6b58b5c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/7111af08da8046bc87246c0a1eaabe6f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/82082002cc1a4418addd47da135e88f2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/3879c8ca35944a1bbdd2b76756897e69, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/a5eb15f90d75481ea504d1cfdbf4044c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/e7ba4822ee0f498cb12705d247a42175, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/7c2bc4ae0bf44779aae55bb24addd759, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/a6564387c00644e2b5e315260953dc79, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/0cb957e89ba849fdbc9035e2c3ec160b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/e07eec0a43584aacb4a2521290bdfec6, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/6a7c342e955645a98d7b54eee4e97881, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/107352098a564fb68c4c98275d542bbb, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/416725ddfc5b44e2abda50bb49e52988, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/713fdf581625425d9a6c7de799773fd6, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/fa1136f073824519a41d7e6935af4819] to archive 2024-11-14T21:32:35,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:32:35,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/6e6807177850408e8bc1d6c2ff772f32 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/6e6807177850408e8bc1d6c2ff772f32 2024-11-14T21:32:35,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/d120c9badf9646b2bf930c103a3dd782 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/d120c9badf9646b2bf930c103a3dd782 2024-11-14T21:32:35,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/59910bfa2439438e8405bd12eafa67ec to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/59910bfa2439438e8405bd12eafa67ec 2024-11-14T21:32:35,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/0e1e307d55b74ddd9480787928d976ae to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/0e1e307d55b74ddd9480787928d976ae 2024-11-14T21:32:35,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/5d5a966caf974a9587ef02aeadb09754 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/5d5a966caf974a9587ef02aeadb09754 2024-11-14T21:32:35,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/2cf98273e79148178341c8358c1a249e to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/2cf98273e79148178341c8358c1a249e 2024-11-14T21:32:35,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/8523ae2d2b9f43eebf5ae51c84188a53 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/8523ae2d2b9f43eebf5ae51c84188a53 2024-11-14T21:32:35,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/8c1d84db42584b8bb000633423971555 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/8c1d84db42584b8bb000633423971555 2024-11-14T21:32:35,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/64a49ec24a83477986c7e6639b027f3d to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/64a49ec24a83477986c7e6639b027f3d 2024-11-14T21:32:35,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/bc79a8d2b436441ab7c6739f01c797ea to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/bc79a8d2b436441ab7c6739f01c797ea 2024-11-14T21:32:35,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/a92f10e1b3bb4a309f8759047b0f8e1b to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/a92f10e1b3bb4a309f8759047b0f8e1b 2024-11-14T21:32:35,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/5a201678e37a4eaa89a301239bfcd989 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/5a201678e37a4eaa89a301239bfcd989 2024-11-14T21:32:35,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/efe9a8669f624b11a27b5c47fe8f4fe3 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/efe9a8669f624b11a27b5c47fe8f4fe3 2024-11-14T21:32:35,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/6b8f1ee2e5c14e04948e524c124200b2 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/6b8f1ee2e5c14e04948e524c124200b2 2024-11-14T21:32:35,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/9b5b4c2a3d2f4ac6ae21dee55f78ecb1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/9b5b4c2a3d2f4ac6ae21dee55f78ecb1 2024-11-14T21:32:35,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/1833257b4e4249d8a061651e7874f9a9 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/1833257b4e4249d8a061651e7874f9a9 2024-11-14T21:32:35,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/50638068572d44f9a3192d230539c628 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/50638068572d44f9a3192d230539c628 2024-11-14T21:32:35,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/2270d20a38394b4f90dbcde54c0399ac to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/2270d20a38394b4f90dbcde54c0399ac 2024-11-14T21:32:35,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/a79c794842404f30812323be2bf22616 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/a79c794842404f30812323be2bf22616 2024-11-14T21:32:35,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/5a0e1eabe218454581b87afba6b58b5c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/5a0e1eabe218454581b87afba6b58b5c 2024-11-14T21:32:35,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/7111af08da8046bc87246c0a1eaabe6f to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/7111af08da8046bc87246c0a1eaabe6f 2024-11-14T21:32:35,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/82082002cc1a4418addd47da135e88f2 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/82082002cc1a4418addd47da135e88f2 2024-11-14T21:32:35,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/3879c8ca35944a1bbdd2b76756897e69 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/3879c8ca35944a1bbdd2b76756897e69 2024-11-14T21:32:35,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/a5eb15f90d75481ea504d1cfdbf4044c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/a5eb15f90d75481ea504d1cfdbf4044c 2024-11-14T21:32:35,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/e7ba4822ee0f498cb12705d247a42175 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/e7ba4822ee0f498cb12705d247a42175 2024-11-14T21:32:35,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/7c2bc4ae0bf44779aae55bb24addd759 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/7c2bc4ae0bf44779aae55bb24addd759 2024-11-14T21:32:35,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/a6564387c00644e2b5e315260953dc79 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/a6564387c00644e2b5e315260953dc79 2024-11-14T21:32:35,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/0cb957e89ba849fdbc9035e2c3ec160b to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/0cb957e89ba849fdbc9035e2c3ec160b 2024-11-14T21:32:35,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/e07eec0a43584aacb4a2521290bdfec6 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/e07eec0a43584aacb4a2521290bdfec6 2024-11-14T21:32:35,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/6a7c342e955645a98d7b54eee4e97881 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/6a7c342e955645a98d7b54eee4e97881 2024-11-14T21:32:35,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/107352098a564fb68c4c98275d542bbb to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/107352098a564fb68c4c98275d542bbb 2024-11-14T21:32:35,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/416725ddfc5b44e2abda50bb49e52988 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/416725ddfc5b44e2abda50bb49e52988 2024-11-14T21:32:35,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/713fdf581625425d9a6c7de799773fd6 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/713fdf581625425d9a6c7de799773fd6 2024-11-14T21:32:35,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/fa1136f073824519a41d7e6935af4819 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/fa1136f073824519a41d7e6935af4819 2024-11-14T21:32:35,337 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/recovered.edits/499.seqid, newMaxSeqId=499, maxSeqId=1 2024-11-14T21:32:35,338 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. 2024-11-14T21:32:35,338 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1676): Region close journal for e9722fc966803fe6d499d80c4851a0fa: Waiting for close lock at 1731619954221Running coprocessor pre-close hooks at 1731619954221Disabling compacts and flushes for region at 1731619954221Disabling writes for close at 1731619954222 (+1 ms)Obtaining lock to block concurrent updates at 1731619954222Preparing flush snapshotting stores in e9722fc966803fe6d499d80c4851a0fa at 1731619954222Finished memstore snapshotting TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731619954222Flushing stores of TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa. at 1731619954223 (+1 ms)Flushing e9722fc966803fe6d499d80c4851a0fa/A: creating writer at 1731619954223Flushing e9722fc966803fe6d499d80c4851a0fa/A: appending metadata at 1731619954226 (+3 ms)Flushing e9722fc966803fe6d499d80c4851a0fa/A: closing flushed file at 1731619954226Flushing e9722fc966803fe6d499d80c4851a0fa/B: creating writer at 1731619954638 (+412 ms)Flushing e9722fc966803fe6d499d80c4851a0fa/B: appending metadata at 1731619954642 (+4 ms)Flushing e9722fc966803fe6d499d80c4851a0fa/B: closing flushed file at 1731619954642Flushing e9722fc966803fe6d499d80c4851a0fa/C: creating writer at 1731619955056 (+414 ms)Flushing e9722fc966803fe6d499d80c4851a0fa/C: appending metadata at 1731619955061 (+5 ms)Flushing e9722fc966803fe6d499d80c4851a0fa/C: closing flushed file at 1731619955061Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45435909: reopening flushed file at 1731619955083 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ec2f02e: reopening flushed file at 1731619955090 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ee40196: reopening flushed file at 1731619955095 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for e9722fc966803fe6d499d80c4851a0fa in 879ms, sequenceid=496, compaction requested=false at 1731619955101 (+6 ms)Writing region close event to WAL at 1731619955332 (+231 ms)Running coprocessor post-close hooks at 1731619955337 (+5 ms)Closed at 1731619955338 (+1 ms) 2024-11-14T21:32:35,342 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(157): Closed e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:35,343 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=e9722fc966803fe6d499d80c4851a0fa, regionState=CLOSED 2024-11-14T21:32:35,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 because future has completed 2024-11-14T21:32:35,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-14T21:32:35,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseRegionProcedure e9722fc966803fe6d499d80c4851a0fa, server=ef788a4cade8,37683,1731619918560 in 1.2830 sec 2024-11-14T21:32:35,350 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-14T21:32:35,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e9722fc966803fe6d499d80c4851a0fa, UNASSIGN in 1.2950 sec 2024-11-14T21:32:35,354 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-14T21:32:35,354 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.3040 sec 2024-11-14T21:32:35,356 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731619955355"}]},"ts":"1731619955355"} 2024-11-14T21:32:35,363 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T21:32:35,363 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T21:32:35,366 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3420 sec 2024-11-14T21:32:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T21:32:36,181 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:32:36,184 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T21:32:36,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:32:36,190 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:32:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T21:32:36,192 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=25, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:32:36,198 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:36,203 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A, FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B, FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C, FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/recovered.edits] 2024-11-14T21:32:36,206 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/71bfc07cc4e144028f3b8fd1c98d11b7 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/71bfc07cc4e144028f3b8fd1c98d11b7 2024-11-14T21:32:36,208 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/b7dde761177f429589b96da62cb34da4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/A/b7dde761177f429589b96da62cb34da4 2024-11-14T21:32:36,211 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/cec05a9782fc4f749d5cc9d362428b49 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/cec05a9782fc4f749d5cc9d362428b49 2024-11-14T21:32:36,213 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/e43038eb68974e848e49a0d0e8098455 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/B/e43038eb68974e848e49a0d0e8098455 2024-11-14T21:32:36,216 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/7aca8eff150c495fb5caecfbbf6afe3d to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/7aca8eff150c495fb5caecfbbf6afe3d 2024-11-14T21:32:36,218 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/e6f5ab5177f64c95a9f7a1e65c648ca1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/C/e6f5ab5177f64c95a9f7a1e65c648ca1 2024-11-14T21:32:36,222 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/recovered.edits/499.seqid to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa/recovered.edits/499.seqid 2024-11-14T21:32:36,223 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/e9722fc966803fe6d499d80c4851a0fa 2024-11-14T21:32:36,223 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T21:32:36,230 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=25, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:32:36,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-14T21:32:36,245 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T21:32:36,285 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T21:32:36,287 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=25, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:32:36,287 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T21:32:36,288 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731619956287"}]},"ts":"9223372036854775807"} 2024-11-14T21:32:36,297 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T21:32:36,297 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e9722fc966803fe6d499d80c4851a0fa, NAME => 'TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T21:32:36,298 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T21:32:36,298 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731619956298"}]},"ts":"9223372036854775807"} 2024-11-14T21:32:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T21:32:36,302 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T21:32:36,304 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=25, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:32:36,305 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 119 msec 2024-11-14T21:32:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T21:32:36,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T21:32:36,512 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:32:36,534 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_967508010_22 at /127.0.0.1:51204 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;ef788a4cade8:37683-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=809 (was 681) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1178 (was 1419) 2024-11-14T21:32:36,554 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=809, ProcessCount=11, AvailableMemoryMB=1177 2024-11-14T21:32:36,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T21:32:36,556 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T21:32:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T21:32:36,563 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T21:32:36,563 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:36,563 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 26 2024-11-14T21:32:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-14T21:32:36,564 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T21:32:36,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741947_1123 (size=1061) 2024-11-14T21:32:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-14T21:32:36,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-14T21:32:36,979 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275 2024-11-14T21:32:36,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741948_1124 (size=53) 2024-11-14T21:32:37,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-14T21:32:37,395 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:32:37,395 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 637fa2090a72f9209083e48908b752fe, disabling compactions & flushes 2024-11-14T21:32:37,395 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:37,395 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:37,395 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. after waiting 0 ms 2024-11-14T21:32:37,395 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:37,395 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:37,395 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 637fa2090a72f9209083e48908b752fe: Waiting for close lock at 1731619957395Disabling compacts and flushes for region at 1731619957395Disabling writes for close at 1731619957395Writing region close event to WAL at 1731619957395Closed at 1731619957395 2024-11-14T21:32:37,397 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T21:32:37,397 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731619957397"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731619957397"}]},"ts":"1731619957397"} 2024-11-14T21:32:37,399 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T21:32:37,400 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T21:32:37,401 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731619957401"}]},"ts":"1731619957401"} 2024-11-14T21:32:37,403 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T21:32:37,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=637fa2090a72f9209083e48908b752fe, ASSIGN}] 2024-11-14T21:32:37,404 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=637fa2090a72f9209083e48908b752fe, ASSIGN 2024-11-14T21:32:37,405 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=637fa2090a72f9209083e48908b752fe, ASSIGN; state=OFFLINE, location=ef788a4cade8,37683,1731619918560; forceNewPlan=false, retain=false 2024-11-14T21:32:37,556 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=637fa2090a72f9209083e48908b752fe, regionState=OPENING, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:37,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=637fa2090a72f9209083e48908b752fe, ASSIGN because future has completed 2024-11-14T21:32:37,560 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560}] 2024-11-14T21:32:37,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-14T21:32:37,726 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:37,727 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:32:37,727 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:37,727 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:32:37,730 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:37,731 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:37,734 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:37,738 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:32:37,739 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 637fa2090a72f9209083e48908b752fe columnFamilyName A 2024-11-14T21:32:37,739 DEBUG [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:37,748 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] regionserver.HStore(327): Store=637fa2090a72f9209083e48908b752fe/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:32:37,749 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:37,759 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:32:37,759 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 637fa2090a72f9209083e48908b752fe columnFamilyName B 2024-11-14T21:32:37,759 DEBUG [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:37,762 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] regionserver.HStore(327): Store=637fa2090a72f9209083e48908b752fe/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:32:37,763 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:37,771 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:32:37,771 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 637fa2090a72f9209083e48908b752fe columnFamilyName C 2024-11-14T21:32:37,771 DEBUG [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:37,774 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] regionserver.HStore(327): Store=637fa2090a72f9209083e48908b752fe/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:32:37,778 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:37,779 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:37,780 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:37,783 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:37,798 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:37,798 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:37,800 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T21:32:37,808 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:37,828 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T21:32:37,830 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened 637fa2090a72f9209083e48908b752fe; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66402589, jitterRate=-0.010524317622184753}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T21:32:37,830 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:37,835 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for 637fa2090a72f9209083e48908b752fe: Running coprocessor pre-open hook at 1731619957731Writing region info on filesystem at 1731619957731Initializing all the Stores at 1731619957732 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731619957733 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731619957734 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731619957734Cleaning up temporary data from old regions at 1731619957799 (+65 ms)Running coprocessor post-open hooks at 1731619957830 (+31 ms)Region opened successfully at 1731619957835 (+5 ms) 2024-11-14T21:32:37,836 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., pid=28, masterSystemTime=1731619957714 2024-11-14T21:32:37,847 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:37,847 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:37,851 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=637fa2090a72f9209083e48908b752fe, regionState=OPEN, openSeqNum=2, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:37,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 because future has completed 2024-11-14T21:32:37,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-14T21:32:37,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 in 303 msec 2024-11-14T21:32:37,873 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T21:32:37,873 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731619957873"}]},"ts":"1731619957873"} 2024-11-14T21:32:37,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-14T21:32:37,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=637fa2090a72f9209083e48908b752fe, ASSIGN in 463 msec 2024-11-14T21:32:37,879 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T21:32:37,880 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T21:32:37,882 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3230 sec 2024-11-14T21:32:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-14T21:32:38,711 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:32:38,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66b4e496, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:38,715 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:32:38,715 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:32:38,723 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:32:38,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:32:38,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:32:38,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33ea22b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:38,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:32:38,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:32:38,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:38,726 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:32:38,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@695908f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:38,728 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:32:38,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:32:38,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:32:38,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:32:38,740 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T21:32:38,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is ef788a4cade8,38255,1731619917503 2024-11-14T21:32:38,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@57da0b84 2024-11-14T21:32:38,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T21:32:38,744 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T21:32:38,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T21:32:38,752 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T21:32:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T21:32:38,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741949_1125 (size=1097) 2024-11-14T21:32:38,803 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-14T21:32:38,803 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-14T21:32:38,808 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T21:32:38,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=637fa2090a72f9209083e48908b752fe, REOPEN/MOVE}] 2024-11-14T21:32:38,831 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=637fa2090a72f9209083e48908b752fe, REOPEN/MOVE 2024-11-14T21:32:38,836 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=637fa2090a72f9209083e48908b752fe, regionState=CLOSING, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:38,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=637fa2090a72f9209083e48908b752fe, REOPEN/MOVE because future has completed 2024-11-14T21:32:38,850 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T21:32:38,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560}] 2024-11-14T21:32:39,024 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:39,024 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T21:32:39,024 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 637fa2090a72f9209083e48908b752fe, disabling compactions & flushes 2024-11-14T21:32:39,024 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:39,024 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:39,024 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. after waiting 0 ms 2024-11-14T21:32:39,024 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:39,029 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T21:32:39,029 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:39,029 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 637fa2090a72f9209083e48908b752fe: Waiting for close lock at 1731619959024Running coprocessor pre-close hooks at 1731619959024Disabling compacts and flushes for region at 1731619959024Disabling writes for close at 1731619959024Writing region close event to WAL at 1731619959024Running coprocessor post-close hooks at 1731619959029 (+5 ms)Closed at 1731619959029 2024-11-14T21:32:39,030 WARN [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionServer(3299): Not adding moved region record: 637fa2090a72f9209083e48908b752fe to self. 2024-11-14T21:32:39,032 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:39,033 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=637fa2090a72f9209083e48908b752fe, regionState=CLOSED 2024-11-14T21:32:39,035 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 because future has completed 2024-11-14T21:32:39,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-14T21:32:39,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 in 187 msec 2024-11-14T21:32:39,041 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=637fa2090a72f9209083e48908b752fe, REOPEN/MOVE; state=CLOSED, location=ef788a4cade8,37683,1731619918560; forceNewPlan=false, retain=true 2024-11-14T21:32:39,192 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=637fa2090a72f9209083e48908b752fe, regionState=OPENING, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:39,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=637fa2090a72f9209083e48908b752fe, REOPEN/MOVE because future has completed 2024-11-14T21:32:39,195 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560}] 2024-11-14T21:32:39,351 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:39,351 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:32:39,352 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:39,352 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:32:39,352 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:39,352 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:39,357 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:39,358 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:32:39,364 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 637fa2090a72f9209083e48908b752fe columnFamilyName A 2024-11-14T21:32:39,367 DEBUG [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:39,367 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] regionserver.HStore(327): Store=637fa2090a72f9209083e48908b752fe/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:32:39,368 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:39,369 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:32:39,369 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 637fa2090a72f9209083e48908b752fe columnFamilyName B 2024-11-14T21:32:39,369 DEBUG [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:39,370 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] regionserver.HStore(327): Store=637fa2090a72f9209083e48908b752fe/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:32:39,370 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:39,370 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:32:39,371 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 637fa2090a72f9209083e48908b752fe columnFamilyName C 2024-11-14T21:32:39,371 DEBUG [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:39,371 INFO [StoreOpener-637fa2090a72f9209083e48908b752fe-1 {}] regionserver.HStore(327): Store=637fa2090a72f9209083e48908b752fe/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:32:39,371 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:39,372 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:39,372 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:39,373 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:39,375 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:39,375 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:39,375 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T21:32:39,377 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:39,378 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 637fa2090a72f9209083e48908b752fe; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62627274, jitterRate=-0.06678089499473572}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T21:32:39,378 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:39,380 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 637fa2090a72f9209083e48908b752fe: Running coprocessor pre-open hook at 1731619959352Writing region info on filesystem at 1731619959352Initializing all the Stores at 1731619959353 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731619959353Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731619959353Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731619959354 (+1 ms)Cleaning up temporary data from old regions at 1731619959375 (+21 ms)Running coprocessor post-open hooks at 1731619959378 (+3 ms)Region opened successfully at 1731619959380 (+2 ms) 2024-11-14T21:32:39,382 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., pid=33, masterSystemTime=1731619959347 2024-11-14T21:32:39,385 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:39,385 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:39,386 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=637fa2090a72f9209083e48908b752fe, regionState=OPEN, openSeqNum=5, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:39,388 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 because future has completed 2024-11-14T21:32:39,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=31 2024-11-14T21:32:39,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 in 195 msec 2024-11-14T21:32:39,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-14T21:32:39,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=637fa2090a72f9209083e48908b752fe, REOPEN/MOVE in 574 msec 2024-11-14T21:32:39,400 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-14T21:32:39,400 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 589 msec 2024-11-14T21:32:39,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 647 msec 2024-11-14T21:32:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-14T21:32:39,415 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:32:39,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77df5202, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:32:39,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:32:39,417 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:32:39,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:32:39,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:32:39,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f7adfaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:32:39,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:32:39,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:39,420 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:32:39,420 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f07f36a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cd952c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,421 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:32:39,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:32:39,422 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:32:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:32:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:32:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3035364d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:32:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:32:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:39,424 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:32:39,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3176774f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf7761, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,426 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:32:39,426 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:32:39,427 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:32:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:32:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:32:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@747d8c4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:32:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:32:39,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:39,428 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:32:39,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b9c8815, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a3f33b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,430 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:32:39,430 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:32:39,431 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:32:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:32:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:32:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b4d5bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:32:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:32:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:39,433 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:32:39,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@573a96c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@465441fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,439 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:32:39,439 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:32:39,440 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:32:39,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:32:39,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:32:39,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d0e1f3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:32:39,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:32:39,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:39,441 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:32:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@88c7143, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70c5ce39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,446 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:32:39,446 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:32:39,447 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:32:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:32:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:32:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dc0c16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:32:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:32:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:39,449 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:32:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8b1075, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f423292, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,453 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:32:39,453 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:32:39,453 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:32:39,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:32:39,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:32:39,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a342f5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:32:39,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:32:39,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:39,455 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:32:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74841f5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,456 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bb3dce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,456 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:32:39,456 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:32:39,457 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:32:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:32:39,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:32:39,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@94a4f83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:32:39,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:32:39,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:39,459 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:32:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4abed2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,462 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f988e58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,463 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:32:39,463 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:32:39,463 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:32:39,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:32:39,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:32:39,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c67949b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:32:39,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:32:39,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:39,465 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:32:39,465 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bb02826, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:32:39,466 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:32:39,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:32:39,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:32:39,468 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:32:39,470 DEBUG [Thread-646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:32:39,471 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:32:39,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:32:39,472 DEBUG [Thread-650 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:32:39,472 DEBUG [Thread-650 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:32:39,473 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:32:39,474 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:32:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:32:39,475 DEBUG [Thread-641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:32:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:32:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:32:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:32:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:32:39,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:32:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:32:39,476 DEBUG [Thread-652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:32:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:32:39,478 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:32:39,478 DEBUG [Thread-643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:32:39,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:32:39,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:32:39,483 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:32:39,483 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:32:39,484 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:32:39,484 DEBUG [Thread-639 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:32:39,484 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:32:39,484 DEBUG [Thread-639 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:32:39,485 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:32:39,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:32:39,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:32:39,493 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:32:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-11-14T21:32:39,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:32:39,496 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:32:39,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:32:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T21:32:39,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:32:39,499 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:32:39,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:32:39,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:32:39,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:32:39,503 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:32:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:32:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:32:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:32:39,510 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:32:39,511 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:32:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:39,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 637fa2090a72f9209083e48908b752fe 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T21:32:39,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=A 2024-11-14T21:32:39,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:39,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=B 2024-11-14T21:32:39,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:39,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=C 2024-11-14T21:32:39,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:39,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ad01cc5033754737b0aefdb720f15d94_637fa2090a72f9209083e48908b752fe is 50, key is test_row_0/A:col10/1731619959522/Put/seqid=0 2024-11-14T21:32:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T21:32:39,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53758 deadline: 1731620019615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:39,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:39,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:39,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:39,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731619959556, max timestamp=1731619959617Num uniques 859; 2024-11-14T21:32:39,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:32:39,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731619959556, max timestamp=1731619959617Num uniques 859; 2024-11-14T21:32:39,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:32:39,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731619959556, max timestamp=1731619959617Num uniques 859; 2024-11-14T21:32:39,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:32:39,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741950_1126 (size=12154) 2024-11-14T21:32:39,637 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:39,650 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ad01cc5033754737b0aefdb720f15d94_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ad01cc5033754737b0aefdb720f15d94_637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:39,652 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/4290bdf939514c79ade90297625292ea, store: [table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:39,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/4290bdf939514c79ade90297625292ea is 175, key is test_row_0/A:col10/1731619959522/Put/seqid=0 2024-11-14T21:32:39,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T21:32:39,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:39,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:39,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:39,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:39,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:39,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:39,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741951_1127 (size=30955) 2024-11-14T21:32:39,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:39,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620019721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53742 deadline: 1731620019721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:39,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53758 deadline: 1731620019731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:39,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1731620019732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:39,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:39,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53780 deadline: 1731620019733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:39,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:39,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:39,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T21:32:39,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T21:32:39,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:39,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:39,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:39,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:39,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:39,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620019871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:39,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:39,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:39,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:39,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53780 deadline: 1731620019872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:39,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53742 deadline: 1731620019872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:39,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1731620019872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:39,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:39,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:39,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:39,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:39,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:39,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:39,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53758 deadline: 1731620019971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:39,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:39,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:39,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:39,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T21:32:39,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:39,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:39,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:39,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:39,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:40,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1731620020091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:40,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:40,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53780 deadline: 1731620020091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:40,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:40,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620020091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:40,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:40,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53742 deadline: 1731620020091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:40,126 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/4290bdf939514c79ade90297625292ea 2024-11-14T21:32:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T21:32:40,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T21:32:40,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:40,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:40,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:40,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:40,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:40,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/796aa3be47e145f18db4bc5950d8b383 is 50, key is test_row_0/B:col10/1731619959522/Put/seqid=0 2024-11-14T21:32:40,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741952_1128 (size=12001) 2024-11-14T21:32:40,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:40,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53758 deadline: 1731620020283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:40,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:40,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T21:32:40,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:40,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:40,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:40,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:40,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:40,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:40,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1731620020401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:40,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:40,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:40,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53780 deadline: 1731620020410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:40,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620020411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:40,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:40,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53742 deadline: 1731620020411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:40,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:40,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:40,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:40,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T21:32:40,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:40,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:40,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:40,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:40,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:40,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:40,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T21:32:40,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:40,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:40,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:40,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:40,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:40,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:40,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T21:32:40,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/796aa3be47e145f18db4bc5950d8b383 2024-11-14T21:32:40,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/88185135041d4ceca00a0db9515cd1db is 50, key is test_row_0/C:col10/1731619959522/Put/seqid=0 2024-11-14T21:32:40,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T21:32:40,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:40,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:40,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:40,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:40,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:40,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53758 deadline: 1731620020801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:40,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741953_1129 (size=12001) 2024-11-14T21:32:40,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1731620020919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:40,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:40,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T21:32:40,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:40,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:40,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:40,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:40,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:40,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:40,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620020971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:40,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53742 deadline: 1731620020972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:40,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:40,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53780 deadline: 1731620020972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:41,013 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T21:32:41,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T21:32:41,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:41,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:41,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:41,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:41,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:41,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:41,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/88185135041d4ceca00a0db9515cd1db 2024-11-14T21:32:41,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/4290bdf939514c79ade90297625292ea as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/4290bdf939514c79ade90297625292ea 2024-11-14T21:32:41,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/4290bdf939514c79ade90297625292ea, entries=150, sequenceid=16, filesize=30.2 K 2024-11-14T21:32:41,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/796aa3be47e145f18db4bc5950d8b383 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/796aa3be47e145f18db4bc5950d8b383 2024-11-14T21:32:41,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/796aa3be47e145f18db4bc5950d8b383, entries=150, sequenceid=16, filesize=11.7 K 2024-11-14T21:32:41,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/88185135041d4ceca00a0db9515cd1db as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/88185135041d4ceca00a0db9515cd1db 2024-11-14T21:32:41,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/88185135041d4ceca00a0db9515cd1db, entries=150, sequenceid=16, filesize=11.7 K 2024-11-14T21:32:41,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 637fa2090a72f9209083e48908b752fe in 1758ms, sequenceid=16, compaction requested=false 2024-11-14T21:32:41,284 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T21:32:41,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:41,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T21:32:41,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:41,302 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 637fa2090a72f9209083e48908b752fe 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-14T21:32:41,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=A 2024-11-14T21:32:41,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:32:41,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=B 2024-11-14T21:32:41,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:32:41,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=C 2024-11-14T21:32:41,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:32:41,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114744cd127dc9a41aea36b5aef97f4c37d_637fa2090a72f9209083e48908b752fe is 50, key is test_row_0/A:col10/1731619959720/Put/seqid=0 2024-11-14T21:32:41,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741954_1130 (size=12154) 2024-11-14T21:32:41,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T21:32:41,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:41,817 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114744cd127dc9a41aea36b5aef97f4c37d_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114744cd127dc9a41aea36b5aef97f4c37d_637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:41,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/195e06f53ee74a938e70c18704c82896, store: [table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:41,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/195e06f53ee74a938e70c18704c82896 is 175, key is test_row_0/A:col10/1731619959720/Put/seqid=0 2024-11-14T21:32:41,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:41,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741955_1131 (size=30955) 2024-11-14T21:32:41,861 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/195e06f53ee74a938e70c18704c82896 2024-11-14T21:32:41,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/382304556f41409ebd3b404ee1181144 is 50, key is test_row_0/B:col10/1731619959720/Put/seqid=0 2024-11-14T21:32:41,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:41,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53758 deadline: 1731620021946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:41,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:41,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:41,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:41,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1731620021960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:41,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741956_1132 (size=12001) 2024-11-14T21:32:42,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620021993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:42,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53742 deadline: 1731620021991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:42,013 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_1' in 637fa2090a72f9209083e48908b752fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2292 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:42,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:42,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:42,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:42,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53780 deadline: 1731620022003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:42,014 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_2' in 637fa2090a72f9209083e48908b752fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2293 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:42,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:42,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:42,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:42,015 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_1' in 637fa2090a72f9209083e48908b752fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2282 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:42,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:42,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:42,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:42,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:42,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1731620022073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53758 deadline: 1731620022073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:42,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:42,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1731620022301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:42,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:42,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53758 deadline: 1731620022301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:42,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:42,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:42,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:42,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:42,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:42,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:42,396 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/382304556f41409ebd3b404ee1181144 2024-11-14T21:32:42,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/5928ea884aba4d1ea1445a10b100acf8 is 50, key is test_row_0/C:col10/1731619959720/Put/seqid=0 2024-11-14T21:32:42,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741957_1133 (size=12001) 2024-11-14T21:32:42,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53758 deadline: 1731620022611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:42,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1731620022620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:42,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:42,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:42,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:42,623 INFO [master/ef788a4cade8:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-14T21:32:42,623 INFO [master/ef788a4cade8:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-14T21:32:42,826 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/5928ea884aba4d1ea1445a10b100acf8 2024-11-14T21:32:42,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/195e06f53ee74a938e70c18704c82896 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/195e06f53ee74a938e70c18704c82896 2024-11-14T21:32:42,887 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/195e06f53ee74a938e70c18704c82896, entries=150, sequenceid=50, filesize=30.2 K 2024-11-14T21:32:42,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/382304556f41409ebd3b404ee1181144 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/382304556f41409ebd3b404ee1181144 2024-11-14T21:32:42,899 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/382304556f41409ebd3b404ee1181144, entries=150, sequenceid=50, filesize=11.7 K 2024-11-14T21:32:42,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/5928ea884aba4d1ea1445a10b100acf8 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/5928ea884aba4d1ea1445a10b100acf8 2024-11-14T21:32:42,935 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/5928ea884aba4d1ea1445a10b100acf8, entries=150, sequenceid=50, filesize=11.7 K 2024-11-14T21:32:42,943 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 637fa2090a72f9209083e48908b752fe in 1641ms, sequenceid=50, compaction requested=false 2024-11-14T21:32:42,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:42,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:42,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-14T21:32:42,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-14T21:32:42,948 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-14T21:32:42,948 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4350 sec 2024-11-14T21:32:42,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 3.4560 sec 2024-11-14T21:32:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:43,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 637fa2090a72f9209083e48908b752fe 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T21:32:43,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=A 2024-11-14T21:32:43,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:43,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=B 2024-11-14T21:32:43,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:43,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=C 2024-11-14T21:32:43,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:43,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114afd14bef5ecd4451bd8e35c5cf815a1a_637fa2090a72f9209083e48908b752fe is 50, key is test_row_0/A:col10/1731619961914/Put/seqid=0 2024-11-14T21:32:43,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741958_1134 (size=14594) 2024-11-14T21:32:43,183 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:43,188 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114afd14bef5ecd4451bd8e35c5cf815a1a_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114afd14bef5ecd4451bd8e35c5cf815a1a_637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:43,190 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/4710fb94d3f347ddb7b95fe9bc41313d, store: [table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:43,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/4710fb94d3f347ddb7b95fe9bc41313d is 175, key is test_row_0/A:col10/1731619961914/Put/seqid=0 2024-11-14T21:32:43,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53758 deadline: 1731620023244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:43,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:43,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:43,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:43,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741959_1135 (size=39549) 2024-11-14T21:32:43,253 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/4710fb94d3f347ddb7b95fe9bc41313d 2024-11-14T21:32:43,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1731620023259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:43,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:43,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:43,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:43,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/d6ecd06f861a4121b3db2c15752bc0d3 is 50, key is test_row_0/B:col10/1731619961914/Put/seqid=0 2024-11-14T21:32:43,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741960_1136 (size=12001) 2024-11-14T21:32:43,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53758 deadline: 1731620023362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:43,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:43,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1731620023381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:43,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:43,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:43,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:43,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:43,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53758 deadline: 1731620023570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:43,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:43,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:43,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:43,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:43,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1731620023594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:43,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:43,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:43,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T21:32:43,679 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:32:43,681 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:32:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-14T21:32:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T21:32:43,704 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:32:43,705 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:32:43,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:32:43,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/d6ecd06f861a4121b3db2c15752bc0d3 2024-11-14T21:32:43,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/875f29448da3434480797e1db2bb54fa is 50, key is test_row_0/C:col10/1731619961914/Put/seqid=0 2024-11-14T21:32:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T21:32:43,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741961_1137 (size=12001) 2024-11-14T21:32:43,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/875f29448da3434480797e1db2bb54fa 2024-11-14T21:32:43,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/4710fb94d3f347ddb7b95fe9bc41313d as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/4710fb94d3f347ddb7b95fe9bc41313d 2024-11-14T21:32:43,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T21:32:43,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:43,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:43,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:43,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:43,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:43,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/4710fb94d3f347ddb7b95fe9bc41313d, entries=200, sequenceid=62, filesize=38.6 K 2024-11-14T21:32:43,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/d6ecd06f861a4121b3db2c15752bc0d3 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/d6ecd06f861a4121b3db2c15752bc0d3 2024-11-14T21:32:43,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/d6ecd06f861a4121b3db2c15752bc0d3, entries=150, sequenceid=62, filesize=11.7 K 2024-11-14T21:32:43,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/875f29448da3434480797e1db2bb54fa as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/875f29448da3434480797e1db2bb54fa 2024-11-14T21:32:43,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53758 deadline: 1731620023915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:43,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:43,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:43,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:43,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1731620023923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:43,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:43,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:43,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:43,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/875f29448da3434480797e1db2bb54fa, entries=150, sequenceid=62, filesize=11.7 K 2024-11-14T21:32:43,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 637fa2090a72f9209083e48908b752fe in 829ms, sequenceid=62, compaction requested=true 2024-11-14T21:32:43,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:43,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 637fa2090a72f9209083e48908b752fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:32:43,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:43,954 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:43,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 637fa2090a72f9209083e48908b752fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:32:43,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:43,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 637fa2090a72f9209083e48908b752fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:32:43,954 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:43,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:43,955 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:43,955 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:43,955 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 637fa2090a72f9209083e48908b752fe/A is initiating minor compaction (all files) 2024-11-14T21:32:43,955 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 637fa2090a72f9209083e48908b752fe/B is initiating minor compaction (all files) 2024-11-14T21:32:43,956 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 637fa2090a72f9209083e48908b752fe/A in TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:43,956 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 637fa2090a72f9209083e48908b752fe/B in TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:43,956 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/4290bdf939514c79ade90297625292ea, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/195e06f53ee74a938e70c18704c82896, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/4710fb94d3f347ddb7b95fe9bc41313d] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp, totalSize=99.1 K 2024-11-14T21:32:43,956 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/796aa3be47e145f18db4bc5950d8b383, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/382304556f41409ebd3b404ee1181144, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/d6ecd06f861a4121b3db2c15752bc0d3] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp, totalSize=35.2 K 2024-11-14T21:32:43,956 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:43,956 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. files: [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/4290bdf939514c79ade90297625292ea, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/195e06f53ee74a938e70c18704c82896, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/4710fb94d3f347ddb7b95fe9bc41313d] 2024-11-14T21:32:43,957 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 796aa3be47e145f18db4bc5950d8b383, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731619959517 2024-11-14T21:32:43,957 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4290bdf939514c79ade90297625292ea, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731619959517 2024-11-14T21:32:43,958 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 195e06f53ee74a938e70c18704c82896, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731619959674 2024-11-14T21:32:43,958 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 382304556f41409ebd3b404ee1181144, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731619959674 2024-11-14T21:32:43,962 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4710fb94d3f347ddb7b95fe9bc41313d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731619961914 2024-11-14T21:32:43,963 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting d6ecd06f861a4121b3db2c15752bc0d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731619961914 2024-11-14T21:32:43,985 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 637fa2090a72f9209083e48908b752fe#B#compaction#119 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:43,985 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/e07da9d63e3d4b1f8d3cbdaf4a44b879 is 50, key is test_row_0/B:col10/1731619961914/Put/seqid=0 2024-11-14T21:32:43,999 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T21:32:44,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T21:32:44,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:44,017 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 637fa2090a72f9209083e48908b752fe 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T21:32:44,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=A 2024-11-14T21:32:44,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:44,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=B 2024-11-14T21:32:44,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:44,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=C 2024-11-14T21:32:44,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:44,020 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111493f65317fc504d8c8d76669774d42715_637fa2090a72f9209083e48908b752fe store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:44,024 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111493f65317fc504d8c8d76669774d42715_637fa2090a72f9209083e48908b752fe, store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:44,024 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111493f65317fc504d8c8d76669774d42715_637fa2090a72f9209083e48908b752fe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:44,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741962_1138 (size=12104) 2024-11-14T21:32:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:44,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:44,042 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/e07da9d63e3d4b1f8d3cbdaf4a44b879 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/e07da9d63e3d4b1f8d3cbdaf4a44b879 2024-11-14T21:32:44,052 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 637fa2090a72f9209083e48908b752fe/B of 637fa2090a72f9209083e48908b752fe into e07da9d63e3d4b1f8d3cbdaf4a44b879(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:44,052 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:44,052 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., storeName=637fa2090a72f9209083e48908b752fe/B, priority=13, startTime=1731619963954; duration=0sec 2024-11-14T21:32:44,053 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:44,053 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 637fa2090a72f9209083e48908b752fe:B 2024-11-14T21:32:44,053 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:44,054 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:44,054 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 637fa2090a72f9209083e48908b752fe/C is initiating minor compaction (all files) 2024-11-14T21:32:44,054 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 637fa2090a72f9209083e48908b752fe/C in TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:44,054 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/88185135041d4ceca00a0db9515cd1db, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/5928ea884aba4d1ea1445a10b100acf8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/875f29448da3434480797e1db2bb54fa] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp, totalSize=35.2 K 2024-11-14T21:32:44,055 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 88185135041d4ceca00a0db9515cd1db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731619959517 2024-11-14T21:32:44,055 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 5928ea884aba4d1ea1445a10b100acf8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731619959674 2024-11-14T21:32:44,057 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 875f29448da3434480797e1db2bb54fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731619961914 2024-11-14T21:32:44,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144c3ebd51337c4485b69e88a10172841a_637fa2090a72f9209083e48908b752fe is 50, key is test_row_0/A:col10/1731619963218/Put/seqid=0 2024-11-14T21:32:44,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53742 deadline: 1731620024058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:44,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620024063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:44,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:44,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:44,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53780 deadline: 1731620024071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:44,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741963_1139 (size=4469) 2024-11-14T21:32:44,112 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 637fa2090a72f9209083e48908b752fe#A#compaction#120 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:44,113 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/c61e45cc046c47a89f9ca95cfff2309f is 175, key is test_row_0/A:col10/1731619961914/Put/seqid=0 2024-11-14T21:32:44,129 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 637fa2090a72f9209083e48908b752fe#C#compaction#122 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:44,130 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/f7b667a1edb64a6c986552b1a4938154 is 50, key is test_row_0/C:col10/1731619961914/Put/seqid=0 2024-11-14T21:32:44,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741965_1141 (size=31058) 2024-11-14T21:32:44,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53742 deadline: 1731620024191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:44,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53780 deadline: 1731620024191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:44,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620024191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:44,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:44,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741964_1140 (size=12154) 2024-11-14T21:32:44,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741966_1142 (size=12104) 2024-11-14T21:32:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T21:32:44,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53742 deadline: 1731620024401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:44,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:44,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:44,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620024410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53780 deadline: 1731620024401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:44,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53758 deadline: 1731620024431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:44,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1731620024442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:44,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:44,618 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/c61e45cc046c47a89f9ca95cfff2309f as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/c61e45cc046c47a89f9ca95cfff2309f 2024-11-14T21:32:44,627 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144c3ebd51337c4485b69e88a10172841a_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144c3ebd51337c4485b69e88a10172841a_637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:44,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/e843be2386274003a341a838deb1a9bb, store: [table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:44,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/e843be2386274003a341a838deb1a9bb is 175, key is test_row_0/A:col10/1731619963218/Put/seqid=0 2024-11-14T21:32:44,653 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 637fa2090a72f9209083e48908b752fe/A of 637fa2090a72f9209083e48908b752fe into c61e45cc046c47a89f9ca95cfff2309f(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:44,653 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:44,653 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., storeName=637fa2090a72f9209083e48908b752fe/A, priority=13, startTime=1731619963954; duration=0sec 2024-11-14T21:32:44,653 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:44,653 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 637fa2090a72f9209083e48908b752fe:A 2024-11-14T21:32:44,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741967_1143 (size=30955) 2024-11-14T21:32:44,686 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/f7b667a1edb64a6c986552b1a4938154 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/f7b667a1edb64a6c986552b1a4938154 2024-11-14T21:32:44,727 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 637fa2090a72f9209083e48908b752fe/C of 637fa2090a72f9209083e48908b752fe into f7b667a1edb64a6c986552b1a4938154(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:44,727 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:44,727 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., storeName=637fa2090a72f9209083e48908b752fe/C, priority=13, startTime=1731619963954; duration=0sec 2024-11-14T21:32:44,728 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:44,728 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 637fa2090a72f9209083e48908b752fe:C 2024-11-14T21:32:44,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53742 deadline: 1731620024725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:44,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:44,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:44,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53780 deadline: 1731620024735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:44,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:44,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:44,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620024735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:44,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:44,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:44,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T21:32:45,083 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/e843be2386274003a341a838deb1a9bb 2024-11-14T21:32:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/8007a6c19753456c99a15b588cb7cc0b is 50, key is test_row_0/B:col10/1731619963218/Put/seqid=0 2024-11-14T21:32:45,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741968_1144 (size=12001) 2024-11-14T21:32:45,136 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/8007a6c19753456c99a15b588cb7cc0b 2024-11-14T21:32:45,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/21d65e9621154d74a92ca92015da04c9 is 50, key is test_row_0/C:col10/1731619963218/Put/seqid=0 2024-11-14T21:32:45,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741969_1145 (size=12001) 2024-11-14T21:32:45,176 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/21d65e9621154d74a92ca92015da04c9 2024-11-14T21:32:45,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/e843be2386274003a341a838deb1a9bb as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/e843be2386274003a341a838deb1a9bb 2024-11-14T21:32:45,218 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/e843be2386274003a341a838deb1a9bb, entries=150, sequenceid=82, filesize=30.2 K 2024-11-14T21:32:45,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/8007a6c19753456c99a15b588cb7cc0b as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/8007a6c19753456c99a15b588cb7cc0b 2024-11-14T21:32:45,241 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/8007a6c19753456c99a15b588cb7cc0b, entries=150, sequenceid=82, filesize=11.7 K 2024-11-14T21:32:45,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:45,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53742 deadline: 1731620025240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:45,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:45,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:45,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/21d65e9621154d74a92ca92015da04c9 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/21d65e9621154d74a92ca92015da04c9 2024-11-14T21:32:45,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:45,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53780 deadline: 1731620025261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:45,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:45,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:45,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:45,272 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/21d65e9621154d74a92ca92015da04c9, entries=150, sequenceid=82, filesize=11.7 K 2024-11-14T21:32:45,279 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 637fa2090a72f9209083e48908b752fe in 1262ms, sequenceid=82, compaction requested=false 2024-11-14T21:32:45,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:45,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:45,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-14T21:32:45,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-14T21:32:45,285 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-14T21:32:45,285 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5770 sec 2024-11-14T21:32:45,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:45,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 637fa2090a72f9209083e48908b752fe 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T21:32:45,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=A 2024-11-14T21:32:45,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:45,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=B 2024-11-14T21:32:45,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:45,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=C 2024-11-14T21:32:45,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:45,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.6080 sec 2024-11-14T21:32:45,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c3190665f4274d9e8d1253cb5ed642f9_637fa2090a72f9209083e48908b752fe is 50, key is test_row_0/A:col10/1731619965273/Put/seqid=0 2024-11-14T21:32:45,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741970_1146 (size=12154) 2024-11-14T21:32:45,325 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:45,331 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c3190665f4274d9e8d1253cb5ed642f9_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c3190665f4274d9e8d1253cb5ed642f9_637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:45,336 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/057caa89e18d41069b3ea7db14293f23, store: [table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:45,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/057caa89e18d41069b3ea7db14293f23 is 175, key is test_row_0/A:col10/1731619965273/Put/seqid=0 2024-11-14T21:32:45,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741971_1147 (size=30955) 2024-11-14T21:32:45,345 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/057caa89e18d41069b3ea7db14293f23 2024-11-14T21:32:45,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/ab34b797974c43d5b6eb7e6f33e61d40 is 50, key is test_row_0/B:col10/1731619965273/Put/seqid=0 2024-11-14T21:32:45,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741972_1148 (size=12001) 2024-11-14T21:32:45,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53758 deadline: 1731620025451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:45,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:45,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620025454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:45,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:45,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:45,457 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_2' in 637fa2090a72f9209083e48908b752fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:45,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1731620025460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:45,473 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_1' in 637fa2090a72f9209083e48908b752fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:45,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:45,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620025560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:45,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:45,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:45,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:45,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620025770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:45,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/ab34b797974c43d5b6eb7e6f33e61d40 2024-11-14T21:32:45,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/e8fbddd381a14832b032bf70e576e127 is 50, key is test_row_0/C:col10/1731619965273/Put/seqid=0 2024-11-14T21:32:45,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741973_1149 (size=12001) 2024-11-14T21:32:45,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T21:32:45,841 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:32:45,842 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:32:45,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-14T21:32:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T21:32:45,847 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:32:45,848 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:32:45,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:32:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T21:32:46,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T21:32:46,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:46,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:46,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:46,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:46,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:46,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:46,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620026095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T21:32:46,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T21:32:46,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:46,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:46,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:46,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:46,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:46,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:46,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/e8fbddd381a14832b032bf70e576e127 2024-11-14T21:32:46,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/057caa89e18d41069b3ea7db14293f23 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/057caa89e18d41069b3ea7db14293f23 2024-11-14T21:32:46,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/057caa89e18d41069b3ea7db14293f23, entries=150, sequenceid=98, filesize=30.2 K 2024-11-14T21:32:46,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/ab34b797974c43d5b6eb7e6f33e61d40 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/ab34b797974c43d5b6eb7e6f33e61d40 2024-11-14T21:32:46,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:46,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53742 deadline: 1731620026251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:46,261 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_2' in 637fa2090a72f9209083e48908b752fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:46,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:46,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:46,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:46,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53780 deadline: 1731620026270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:46,275 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_0' in 637fa2090a72f9209083e48908b752fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:46,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:46,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:46,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:46,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/ab34b797974c43d5b6eb7e6f33e61d40, entries=150, sequenceid=98, filesize=11.7 K 2024-11-14T21:32:46,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/e8fbddd381a14832b032bf70e576e127 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/e8fbddd381a14832b032bf70e576e127 2024-11-14T21:32:46,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/e8fbddd381a14832b032bf70e576e127, entries=150, sequenceid=98, filesize=11.7 K 2024-11-14T21:32:46,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 637fa2090a72f9209083e48908b752fe in 1026ms, sequenceid=98, compaction requested=true 2024-11-14T21:32:46,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:46,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 637fa2090a72f9209083e48908b752fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:32:46,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:46,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 637fa2090a72f9209083e48908b752fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:32:46,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:32:46,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 637fa2090a72f9209083e48908b752fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:32:46,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T21:32:46,312 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:46,312 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:46,315 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:46,320 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:46,320 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 637fa2090a72f9209083e48908b752fe/C is initiating minor compaction (all files) 2024-11-14T21:32:46,320 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 637fa2090a72f9209083e48908b752fe/C in TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:46,320 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/f7b667a1edb64a6c986552b1a4938154, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/21d65e9621154d74a92ca92015da04c9, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/e8fbddd381a14832b032bf70e576e127] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp, totalSize=35.3 K 2024-11-14T21:32:46,320 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 637fa2090a72f9209083e48908b752fe/A is initiating minor compaction (all files) 2024-11-14T21:32:46,320 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 637fa2090a72f9209083e48908b752fe/A in TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:46,320 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/c61e45cc046c47a89f9ca95cfff2309f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/e843be2386274003a341a838deb1a9bb, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/057caa89e18d41069b3ea7db14293f23] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp, totalSize=90.8 K 2024-11-14T21:32:46,321 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:46,321 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. files: [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/c61e45cc046c47a89f9ca95cfff2309f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/e843be2386274003a341a838deb1a9bb, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/057caa89e18d41069b3ea7db14293f23] 2024-11-14T21:32:46,321 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting f7b667a1edb64a6c986552b1a4938154, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731619961914 2024-11-14T21:32:46,322 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting c61e45cc046c47a89f9ca95cfff2309f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731619961914 2024-11-14T21:32:46,322 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 21d65e9621154d74a92ca92015da04c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731619963218 2024-11-14T21:32:46,322 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting e843be2386274003a341a838deb1a9bb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731619963218 2024-11-14T21:32:46,322 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting e8fbddd381a14832b032bf70e576e127, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731619964046 2024-11-14T21:32:46,323 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 057caa89e18d41069b3ea7db14293f23, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731619964046 2024-11-14T21:32:46,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T21:32:46,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:46,332 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 637fa2090a72f9209083e48908b752fe 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T21:32:46,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=A 2024-11-14T21:32:46,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:46,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=B 2024-11-14T21:32:46,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:46,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=C 2024-11-14T21:32:46,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:46,338 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 637fa2090a72f9209083e48908b752fe#C#compaction#128 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:46,339 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/d48c8792a65546f6902601a3dc256036 is 50, key is test_row_0/C:col10/1731619965273/Put/seqid=0 2024-11-14T21:32:46,347 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:46,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144e2037c6bd1041ab923dfe17361f3ef7_637fa2090a72f9209083e48908b752fe is 50, key is test_row_0/A:col10/1731619965450/Put/seqid=0 2024-11-14T21:32:46,379 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111426bda97a378a40159804e1e430aa812a_637fa2090a72f9209083e48908b752fe store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:46,382 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111426bda97a378a40159804e1e430aa812a_637fa2090a72f9209083e48908b752fe, store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:46,384 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111426bda97a378a40159804e1e430aa812a_637fa2090a72f9209083e48908b752fe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:46,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741974_1150 (size=12207) 2024-11-14T21:32:46,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741975_1151 (size=12154) 2024-11-14T21:32:46,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:46,448 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144e2037c6bd1041ab923dfe17361f3ef7_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144e2037c6bd1041ab923dfe17361f3ef7_637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:46,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/7360d333eebb4b3fa609ac5c8649c123, store: [table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:46,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/7360d333eebb4b3fa609ac5c8649c123 is 175, key is test_row_0/A:col10/1731619965450/Put/seqid=0 2024-11-14T21:32:46,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741976_1152 (size=4469) 2024-11-14T21:32:46,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741977_1153 (size=30955) 2024-11-14T21:32:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T21:32:46,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:46,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620026776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:46,833 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/d48c8792a65546f6902601a3dc256036 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/d48c8792a65546f6902601a3dc256036 2024-11-14T21:32:46,863 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 637fa2090a72f9209083e48908b752fe/C of 637fa2090a72f9209083e48908b752fe into d48c8792a65546f6902601a3dc256036(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:46,863 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:46,863 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., storeName=637fa2090a72f9209083e48908b752fe/C, priority=13, startTime=1731619966312; duration=0sec 2024-11-14T21:32:46,865 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:46,865 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 637fa2090a72f9209083e48908b752fe:C 2024-11-14T21:32:46,865 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:46,870 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:46,870 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 637fa2090a72f9209083e48908b752fe/B is initiating minor compaction (all files) 2024-11-14T21:32:46,871 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 637fa2090a72f9209083e48908b752fe/B in TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:46,871 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/e07da9d63e3d4b1f8d3cbdaf4a44b879, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/8007a6c19753456c99a15b588cb7cc0b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/ab34b797974c43d5b6eb7e6f33e61d40] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp, totalSize=35.3 K 2024-11-14T21:32:46,872 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting e07da9d63e3d4b1f8d3cbdaf4a44b879, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731619961914 2024-11-14T21:32:46,872 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/7360d333eebb4b3fa609ac5c8649c123 2024-11-14T21:32:46,878 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 637fa2090a72f9209083e48908b752fe#A#compaction#129 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:46,878 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 8007a6c19753456c99a15b588cb7cc0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731619963218 2024-11-14T21:32:46,879 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/161963451ed04eb2a5b40e3e72bd3685 is 175, key is test_row_0/A:col10/1731619965273/Put/seqid=0 2024-11-14T21:32:46,882 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting ab34b797974c43d5b6eb7e6f33e61d40, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731619964046 2024-11-14T21:32:46,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:46,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620026901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:46,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741978_1154 (size=31161) 2024-11-14T21:32:46,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/1d726febb1e1436a8f02cf33f7bbaba8 is 50, key is test_row_0/B:col10/1731619965450/Put/seqid=0 2024-11-14T21:32:46,934 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/161963451ed04eb2a5b40e3e72bd3685 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/161963451ed04eb2a5b40e3e72bd3685 2024-11-14T21:32:46,937 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 637fa2090a72f9209083e48908b752fe#B#compaction#132 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:46,939 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/2c598bfcd8a84679852b893f50bc3204 is 50, key is test_row_0/B:col10/1731619965273/Put/seqid=0 2024-11-14T21:32:46,967 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 637fa2090a72f9209083e48908b752fe/A of 637fa2090a72f9209083e48908b752fe into 161963451ed04eb2a5b40e3e72bd3685(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:46,967 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:46,967 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., storeName=637fa2090a72f9209083e48908b752fe/A, priority=13, startTime=1731619966311; duration=0sec 2024-11-14T21:32:46,967 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:46,967 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 637fa2090a72f9209083e48908b752fe:A 2024-11-14T21:32:46,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T21:32:47,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741979_1155 (size=12001) 2024-11-14T21:32:47,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741980_1156 (size=12207) 2024-11-14T21:32:47,033 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/2c598bfcd8a84679852b893f50bc3204 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/2c598bfcd8a84679852b893f50bc3204 2024-11-14T21:32:47,041 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 637fa2090a72f9209083e48908b752fe/B of 637fa2090a72f9209083e48908b752fe into 2c598bfcd8a84679852b893f50bc3204(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:47,041 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:47,041 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., storeName=637fa2090a72f9209083e48908b752fe/B, priority=13, startTime=1731619966311; duration=0sec 2024-11-14T21:32:47,041 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:47,042 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 637fa2090a72f9209083e48908b752fe:B 2024-11-14T21:32:47,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:47,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620027120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:47,407 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/1d726febb1e1436a8f02cf33f7bbaba8 2024-11-14T21:32:47,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/9130a4fb1d3646e4ba81bf71fd146dab is 50, key is test_row_0/C:col10/1731619965450/Put/seqid=0 2024-11-14T21:32:47,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620027446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:47,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53758 deadline: 1731620027470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:47,480 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_2' in 637fa2090a72f9209083e48908b752fe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:47,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741981_1157 (size=12001) 2024-11-14T21:32:47,484 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/9130a4fb1d3646e4ba81bf71fd146dab 2024-11-14T21:32:47,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/7360d333eebb4b3fa609ac5c8649c123 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/7360d333eebb4b3fa609ac5c8649c123 2024-11-14T21:32:47,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1731620027491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:47,495 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_1' in 637fa2090a72f9209083e48908b752fe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:47,499 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/7360d333eebb4b3fa609ac5c8649c123, entries=150, sequenceid=117, filesize=30.2 K 2024-11-14T21:32:47,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/1d726febb1e1436a8f02cf33f7bbaba8 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/1d726febb1e1436a8f02cf33f7bbaba8 2024-11-14T21:32:47,509 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/1d726febb1e1436a8f02cf33f7bbaba8, entries=150, sequenceid=117, filesize=11.7 K 2024-11-14T21:32:47,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/9130a4fb1d3646e4ba81bf71fd146dab as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/9130a4fb1d3646e4ba81bf71fd146dab 2024-11-14T21:32:47,517 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/9130a4fb1d3646e4ba81bf71fd146dab, entries=150, sequenceid=117, filesize=11.7 K 2024-11-14T21:32:47,527 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 637fa2090a72f9209083e48908b752fe in 1196ms, sequenceid=117, compaction requested=false 2024-11-14T21:32:47,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:47,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:47,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-14T21:32:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-14T21:32:47,537 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-14T21:32:47,537 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6850 sec 2024-11-14T21:32:47,540 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.6960 sec 2024-11-14T21:32:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:47,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 637fa2090a72f9209083e48908b752fe 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T21:32:47,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=A 2024-11-14T21:32:47,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:47,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=B 2024-11-14T21:32:47,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:47,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=C 2024-11-14T21:32:47,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T21:32:48,002 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:32:48,003 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:32:48,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-14T21:32:48,006 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:32:48,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T21:32:48,009 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:32:48,009 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:32:48,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143f5de4f770744b279b315844ecbde134_637fa2090a72f9209083e48908b752fe is 50, key is test_row_0/A:col10/1731619966711/Put/seqid=0 2024-11-14T21:32:48,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741982_1158 (size=12204) 2024-11-14T21:32:48,075 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:48,080 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143f5de4f770744b279b315844ecbde134_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143f5de4f770744b279b315844ecbde134_637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:48,082 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/56a2d7c03fa4463bb03fea8b2cf07f2e, store: [table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:48,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/56a2d7c03fa4463bb03fea8b2cf07f2e is 175, key is test_row_0/A:col10/1731619966711/Put/seqid=0 2024-11-14T21:32:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T21:32:48,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620028127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:48,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:48,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:48,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:48,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741983_1159 (size=31005) 2024-11-14T21:32:48,149 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/56a2d7c03fa4463bb03fea8b2cf07f2e 2024-11-14T21:32:48,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T21:32:48,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:48,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:48,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:48,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:48,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:48,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:48,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/1f2e848cb1154555b51fa82b64f2c243 is 50, key is test_row_0/B:col10/1731619966711/Put/seqid=0 2024-11-14T21:32:48,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741984_1160 (size=12051) 2024-11-14T21:32:48,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/1f2e848cb1154555b51fa82b64f2c243 2024-11-14T21:32:48,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620028246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:48,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:48,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:48,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53742 deadline: 1731620028280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:48,284 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_2' in 637fa2090a72f9209083e48908b752fe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:48,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53780 deadline: 1731620028290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:48,296 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_0' in 637fa2090a72f9209083e48908b752fe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:48,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:48,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:48,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:48,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/b7cd361413f34fdbbe1722194ed95de6 is 50, key is test_row_0/C:col10/1731619966711/Put/seqid=0 2024-11-14T21:32:48,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741985_1161 (size=12051) 2024-11-14T21:32:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T21:32:48,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T21:32:48,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:48,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:48,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:48,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:48,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:48,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620028462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:48,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T21:32:48,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:48,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:48,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:48,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:48,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T21:32:48,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T21:32:48,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:48,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:48,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:48,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:48,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:48,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:48,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/b7cd361413f34fdbbe1722194ed95de6 2024-11-14T21:32:48,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/56a2d7c03fa4463bb03fea8b2cf07f2e as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/56a2d7c03fa4463bb03fea8b2cf07f2e 2024-11-14T21:32:48,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/56a2d7c03fa4463bb03fea8b2cf07f2e, entries=150, sequenceid=134, filesize=30.3 K 2024-11-14T21:32:48,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/1f2e848cb1154555b51fa82b64f2c243 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/1f2e848cb1154555b51fa82b64f2c243 2024-11-14T21:32:48,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/1f2e848cb1154555b51fa82b64f2c243, entries=150, sequenceid=134, filesize=11.8 K 2024-11-14T21:32:48,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/b7cd361413f34fdbbe1722194ed95de6 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/b7cd361413f34fdbbe1722194ed95de6 2024-11-14T21:32:48,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/b7cd361413f34fdbbe1722194ed95de6, entries=150, sequenceid=134, filesize=11.8 K 2024-11-14T21:32:48,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 637fa2090a72f9209083e48908b752fe in 800ms, sequenceid=134, compaction requested=true 2024-11-14T21:32:48,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:48,775 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:48,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 637fa2090a72f9209083e48908b752fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:32:48,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:48,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 637fa2090a72f9209083e48908b752fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:32:48,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:48,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 637fa2090a72f9209083e48908b752fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:32:48,780 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:48,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:48,782 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:48,782 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 637fa2090a72f9209083e48908b752fe/B is initiating minor compaction (all files) 2024-11-14T21:32:48,782 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93121 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:48,782 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 637fa2090a72f9209083e48908b752fe/B in TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:48,782 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 637fa2090a72f9209083e48908b752fe/A is initiating minor compaction (all files) 2024-11-14T21:32:48,782 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/2c598bfcd8a84679852b893f50bc3204, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/1d726febb1e1436a8f02cf33f7bbaba8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/1f2e848cb1154555b51fa82b64f2c243] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp, totalSize=35.4 K 2024-11-14T21:32:48,782 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 637fa2090a72f9209083e48908b752fe/A in TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:48,782 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/161963451ed04eb2a5b40e3e72bd3685, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/7360d333eebb4b3fa609ac5c8649c123, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/56a2d7c03fa4463bb03fea8b2cf07f2e] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp, totalSize=90.9 K 2024-11-14T21:32:48,782 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:48,782 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. files: [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/161963451ed04eb2a5b40e3e72bd3685, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/7360d333eebb4b3fa609ac5c8649c123, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/56a2d7c03fa4463bb03fea8b2cf07f2e] 2024-11-14T21:32:48,784 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c598bfcd8a84679852b893f50bc3204, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731619964046 2024-11-14T21:32:48,784 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 161963451ed04eb2a5b40e3e72bd3685, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731619964046 2024-11-14T21:32:48,784 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d726febb1e1436a8f02cf33f7bbaba8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731619965422 2024-11-14T21:32:48,784 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7360d333eebb4b3fa609ac5c8649c123, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731619965422 2024-11-14T21:32:48,785 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f2e848cb1154555b51fa82b64f2c243, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731619966711 2024-11-14T21:32:48,785 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56a2d7c03fa4463bb03fea8b2cf07f2e, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731619966711 2024-11-14T21:32:48,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 637fa2090a72f9209083e48908b752fe 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T21:32:48,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=A 2024-11-14T21:32:48,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:48,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=B 2024-11-14T21:32:48,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:48,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=C 2024-11-14T21:32:48,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:48,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:48,804 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:48,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T21:32:48,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:48,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:48,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:48,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:48,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:48,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:48,814 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 637fa2090a72f9209083e48908b752fe#B#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:48,814 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/aeab510c920d424faba4f41bf6f73ae4 is 50, key is test_row_0/B:col10/1731619966711/Put/seqid=0 2024-11-14T21:32:48,816 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411142aea49d7f97945b3a70e3c31183d6d76_637fa2090a72f9209083e48908b752fe store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:48,820 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411142aea49d7f97945b3a70e3c31183d6d76_637fa2090a72f9209083e48908b752fe, store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:48,821 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142aea49d7f97945b3a70e3c31183d6d76_637fa2090a72f9209083e48908b752fe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:48,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148059238a53fc4b25a02795b3e9e2394e_637fa2090a72f9209083e48908b752fe is 50, key is test_row_0/A:col10/1731619968781/Put/seqid=0 2024-11-14T21:32:48,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741986_1162 (size=12359) 2024-11-14T21:32:48,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741987_1163 (size=4469) 2024-11-14T21:32:48,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620028869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:48,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:48,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:48,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:48,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741988_1164 (size=14794) 2024-11-14T21:32:48,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T21:32:48,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:48,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:48,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:48,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:48,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:48,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620028991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:49,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T21:32:49,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:49,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:49,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:49,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:49,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:49,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:49,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T21:32:49,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:49,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620029203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:49,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:49,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:49,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:49,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T21:32:49,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:49,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:49,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:49,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:49,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:49,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:49,278 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 637fa2090a72f9209083e48908b752fe#A#compaction#137 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:49,279 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/b85624253df64cc5ad9ed630c7850e25 is 175, key is test_row_0/A:col10/1731619966711/Put/seqid=0 2024-11-14T21:32:49,287 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/aeab510c920d424faba4f41bf6f73ae4 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/aeab510c920d424faba4f41bf6f73ae4 2024-11-14T21:32:49,296 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:49,306 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 637fa2090a72f9209083e48908b752fe/B of 637fa2090a72f9209083e48908b752fe into aeab510c920d424faba4f41bf6f73ae4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:49,306 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:49,306 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., storeName=637fa2090a72f9209083e48908b752fe/B, priority=13, startTime=1731619968780; duration=0sec 2024-11-14T21:32:49,306 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:49,306 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 637fa2090a72f9209083e48908b752fe:B 2024-11-14T21:32:49,306 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:49,308 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:49,308 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 637fa2090a72f9209083e48908b752fe/C is initiating minor compaction (all files) 2024-11-14T21:32:49,308 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 637fa2090a72f9209083e48908b752fe/C in TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:49,308 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/d48c8792a65546f6902601a3dc256036, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/9130a4fb1d3646e4ba81bf71fd146dab, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/b7cd361413f34fdbbe1722194ed95de6] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp, totalSize=35.4 K 2024-11-14T21:32:49,309 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting d48c8792a65546f6902601a3dc256036, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731619964046 2024-11-14T21:32:49,311 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148059238a53fc4b25a02795b3e9e2394e_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148059238a53fc4b25a02795b3e9e2394e_637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:49,319 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 9130a4fb1d3646e4ba81bf71fd146dab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731619965422 2024-11-14T21:32:49,323 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting b7cd361413f34fdbbe1722194ed95de6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731619966711 2024-11-14T21:32:49,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741989_1165 (size=31313) 2024-11-14T21:32:49,332 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/c4925440b8b84e138e3fbe22f63fda64, store: [table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:49,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/c4925440b8b84e138e3fbe22f63fda64 is 175, key is test_row_0/A:col10/1731619968781/Put/seqid=0 2024-11-14T21:32:49,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741990_1166 (size=39749) 2024-11-14T21:32:49,371 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/c4925440b8b84e138e3fbe22f63fda64 2024-11-14T21:32:49,373 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 637fa2090a72f9209083e48908b752fe#C#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:49,373 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/bf92c06cd4504ef6a392eb62df1c7c08 is 50, key is test_row_0/C:col10/1731619966711/Put/seqid=0 2024-11-14T21:32:49,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/d711f2d085cd4a54a56f243a6c2bad65 is 50, key is test_row_0/B:col10/1731619968781/Put/seqid=0 2024-11-14T21:32:49,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741991_1167 (size=12359) 2024-11-14T21:32:49,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741992_1168 (size=12151) 2024-11-14T21:32:49,421 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/bf92c06cd4504ef6a392eb62df1c7c08 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/bf92c06cd4504ef6a392eb62df1c7c08 2024-11-14T21:32:49,426 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 637fa2090a72f9209083e48908b752fe/C of 637fa2090a72f9209083e48908b752fe into bf92c06cd4504ef6a392eb62df1c7c08(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:49,426 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:49,426 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., storeName=637fa2090a72f9209083e48908b752fe/C, priority=13, startTime=1731619968780; duration=0sec 2024-11-14T21:32:49,426 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:49,427 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 637fa2090a72f9209083e48908b752fe:C 2024-11-14T21:32:49,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T21:32:49,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:49,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:49,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:49,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:49,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:49,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620029510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:49,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:49,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:49,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:49,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T21:32:49,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:49,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:49,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:49,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:49,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:49,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T21:32:49,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:49,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:49,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:49,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:49,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:49,743 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/b85624253df64cc5ad9ed630c7850e25 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/b85624253df64cc5ad9ed630c7850e25 2024-11-14T21:32:49,752 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 637fa2090a72f9209083e48908b752fe/A of 637fa2090a72f9209083e48908b752fe into b85624253df64cc5ad9ed630c7850e25(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:49,752 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:49,752 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., storeName=637fa2090a72f9209083e48908b752fe/A, priority=13, startTime=1731619968775; duration=0sec 2024-11-14T21:32:49,752 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:49,752 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 637fa2090a72f9209083e48908b752fe:A 2024-11-14T21:32:49,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/d711f2d085cd4a54a56f243a6c2bad65 2024-11-14T21:32:49,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/ae20f27606a34cea88e7f20375ee8566 is 50, key is test_row_0/C:col10/1731619968781/Put/seqid=0 2024-11-14T21:32:49,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T21:32:49,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:49,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:49,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:49,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:49,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:49,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741993_1169 (size=12151) 2024-11-14T21:32:50,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620030023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:50,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:50,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T21:32:50,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:50,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:50,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:50,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:50,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T21:32:50,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T21:32:50,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:50,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:50,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:50,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:50,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:50,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:50,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/ae20f27606a34cea88e7f20375ee8566 2024-11-14T21:32:50,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/c4925440b8b84e138e3fbe22f63fda64 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/c4925440b8b84e138e3fbe22f63fda64 2024-11-14T21:32:50,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/c4925440b8b84e138e3fbe22f63fda64, entries=200, sequenceid=153, filesize=38.8 K 2024-11-14T21:32:50,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/d711f2d085cd4a54a56f243a6c2bad65 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/d711f2d085cd4a54a56f243a6c2bad65 2024-11-14T21:32:50,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T21:32:50,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:50,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:50,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:50,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:50,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:50,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/d711f2d085cd4a54a56f243a6c2bad65, entries=150, sequenceid=153, filesize=11.9 K 2024-11-14T21:32:50,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/ae20f27606a34cea88e7f20375ee8566 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/ae20f27606a34cea88e7f20375ee8566 2024-11-14T21:32:50,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:50,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/ae20f27606a34cea88e7f20375ee8566, entries=150, sequenceid=153, filesize=11.9 K 2024-11-14T21:32:50,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 637fa2090a72f9209083e48908b752fe in 1587ms, sequenceid=153, compaction requested=false 2024-11-14T21:32:50,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:50,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T21:32:50,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:50,526 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 637fa2090a72f9209083e48908b752fe 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T21:32:50,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=A 2024-11-14T21:32:50,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:50,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=B 2024-11-14T21:32:50,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:50,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=C 2024-11-14T21:32:50,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:50,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ae28d81d35b945e585aff8cbc4e21e17_637fa2090a72f9209083e48908b752fe is 50, key is test_row_0/A:col10/1731619968866/Put/seqid=0 2024-11-14T21:32:50,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741994_1170 (size=12304) 2024-11-14T21:32:50,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:50,989 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ae28d81d35b945e585aff8cbc4e21e17_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ae28d81d35b945e585aff8cbc4e21e17_637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:51,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/9c51e41421984860ae4f90890d080d37, store: [table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:51,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/9c51e41421984860ae4f90890d080d37 is 175, key is test_row_0/A:col10/1731619968866/Put/seqid=0 2024-11-14T21:32:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:51,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:51,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741995_1171 (size=31105) 2024-11-14T21:32:51,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620031168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:51,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620031281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:51,451 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/9c51e41421984860ae4f90890d080d37 2024-11-14T21:32:51,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/a392c672d5fa43bc99c0ba468de7441d is 50, key is test_row_0/B:col10/1731619968866/Put/seqid=0 2024-11-14T21:32:51,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741996_1172 (size=12151) 2024-11-14T21:32:51,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620031491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:51,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:51,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:51,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:51,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53758 deadline: 1731620031501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:51,504 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_2' in 637fa2090a72f9209083e48908b752fe of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:51,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53788 deadline: 1731620031540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:51,541 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_1' in 637fa2090a72f9209083e48908b752fe of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8282 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:51,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620031807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:51,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:51,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:51,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:51,874 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/a392c672d5fa43bc99c0ba468de7441d 2024-11-14T21:32:51,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/9d8e6571a3624ee3aa11980a8808e9c3 is 50, key is test_row_0/C:col10/1731619968866/Put/seqid=0 2024-11-14T21:32:51,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741997_1173 (size=12151) 2024-11-14T21:32:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T21:32:52,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53780 deadline: 1731620032301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:52,303 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_0' in 637fa2090a72f9209083e48908b752fe of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:52,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53742 deadline: 1731620032311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:52,313 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_2' in 637fa2090a72f9209083e48908b752fe of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:32:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:52,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620032321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:52,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:52,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:52,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:52,337 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/9d8e6571a3624ee3aa11980a8808e9c3 2024-11-14T21:32:52,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/9c51e41421984860ae4f90890d080d37 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/9c51e41421984860ae4f90890d080d37 2024-11-14T21:32:52,383 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/9c51e41421984860ae4f90890d080d37, entries=150, sequenceid=169, filesize=30.4 K 2024-11-14T21:32:52,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/a392c672d5fa43bc99c0ba468de7441d as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/a392c672d5fa43bc99c0ba468de7441d 2024-11-14T21:32:52,415 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/a392c672d5fa43bc99c0ba468de7441d, entries=150, sequenceid=169, filesize=11.9 K 2024-11-14T21:32:52,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/9d8e6571a3624ee3aa11980a8808e9c3 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/9d8e6571a3624ee3aa11980a8808e9c3 2024-11-14T21:32:52,424 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/9d8e6571a3624ee3aa11980a8808e9c3, entries=150, sequenceid=169, filesize=11.9 K 2024-11-14T21:32:52,433 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 637fa2090a72f9209083e48908b752fe in 1906ms, sequenceid=169, compaction requested=true 2024-11-14T21:32:52,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:52,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:52,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-14T21:32:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-14T21:32:52,454 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-14T21:32:52,454 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.4410 sec 2024-11-14T21:32:52,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 4.4520 sec 2024-11-14T21:32:53,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:53,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 637fa2090a72f9209083e48908b752fe 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T21:32:53,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=A 2024-11-14T21:32:53,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:53,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=B 2024-11-14T21:32:53,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:53,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=C 2024-11-14T21:32:53,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:53,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f352bbc5bea4426ebd6e54c3f6fe0f5c_637fa2090a72f9209083e48908b752fe is 50, key is test_row_0/A:col10/1731619971164/Put/seqid=0 2024-11-14T21:32:53,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741998_1174 (size=14794) 2024-11-14T21:32:53,380 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:53,389 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f352bbc5bea4426ebd6e54c3f6fe0f5c_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f352bbc5bea4426ebd6e54c3f6fe0f5c_637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:53,398 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/96cedf9e6c5d43758fa5d2a3815faffe, store: [table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:53,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/96cedf9e6c5d43758fa5d2a3815faffe is 175, key is test_row_0/A:col10/1731619971164/Put/seqid=0 2024-11-14T21:32:53,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:53,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620033395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:53,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741999_1175 (size=39749) 2024-11-14T21:32:53,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:53,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620033513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:53,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620033721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:53,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:53,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:53,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:53,827 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/96cedf9e6c5d43758fa5d2a3815faffe 2024-11-14T21:32:53,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/2dc60a577e6d490c9fc75ed7a6751e91 is 50, key is test_row_0/B:col10/1731619971164/Put/seqid=0 2024-11-14T21:32:53,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742000_1176 (size=12151) 2024-11-14T21:32:53,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/2dc60a577e6d490c9fc75ed7a6751e91 2024-11-14T21:32:53,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/aa01ef10bf964156adb19ce62f7e1c1c is 50, key is test_row_0/C:col10/1731619971164/Put/seqid=0 2024-11-14T21:32:53,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742001_1177 (size=12151) 2024-11-14T21:32:53,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/aa01ef10bf964156adb19ce62f7e1c1c 2024-11-14T21:32:54,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/96cedf9e6c5d43758fa5d2a3815faffe as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/96cedf9e6c5d43758fa5d2a3815faffe 2024-11-14T21:32:54,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/96cedf9e6c5d43758fa5d2a3815faffe, entries=200, sequenceid=189, filesize=38.8 K 2024-11-14T21:32:54,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/2dc60a577e6d490c9fc75ed7a6751e91 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/2dc60a577e6d490c9fc75ed7a6751e91 2024-11-14T21:32:54,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/2dc60a577e6d490c9fc75ed7a6751e91, entries=150, sequenceid=189, filesize=11.9 K 2024-11-14T21:32:54,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/aa01ef10bf964156adb19ce62f7e1c1c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/aa01ef10bf964156adb19ce62f7e1c1c 2024-11-14T21:32:54,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620034033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:54,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:54,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:54,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:54,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/aa01ef10bf964156adb19ce62f7e1c1c, entries=150, sequenceid=189, filesize=11.9 K 2024-11-14T21:32:54,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 637fa2090a72f9209083e48908b752fe in 739ms, sequenceid=189, compaction requested=true 2024-11-14T21:32:54,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:54,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 637fa2090a72f9209083e48908b752fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:32:54,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:54,075 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:32:54,075 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:32:54,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 637fa2090a72f9209083e48908b752fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:32:54,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:54,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 637fa2090a72f9209083e48908b752fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:32:54,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:54,082 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:32:54,082 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 637fa2090a72f9209083e48908b752fe/B is initiating minor compaction (all files) 2024-11-14T21:32:54,083 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 637fa2090a72f9209083e48908b752fe/B in TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:54,083 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/aeab510c920d424faba4f41bf6f73ae4, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/d711f2d085cd4a54a56f243a6c2bad65, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/a392c672d5fa43bc99c0ba468de7441d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/2dc60a577e6d490c9fc75ed7a6751e91] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp, totalSize=47.7 K 2024-11-14T21:32:54,083 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141916 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:32:54,083 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 637fa2090a72f9209083e48908b752fe/A is initiating minor compaction (all files) 2024-11-14T21:32:54,083 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 637fa2090a72f9209083e48908b752fe/A in TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:54,083 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/b85624253df64cc5ad9ed630c7850e25, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/c4925440b8b84e138e3fbe22f63fda64, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/9c51e41421984860ae4f90890d080d37, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/96cedf9e6c5d43758fa5d2a3815faffe] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp, totalSize=138.6 K 2024-11-14T21:32:54,083 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:54,083 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. files: [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/b85624253df64cc5ad9ed630c7850e25, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/c4925440b8b84e138e3fbe22f63fda64, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/9c51e41421984860ae4f90890d080d37, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/96cedf9e6c5d43758fa5d2a3815faffe] 2024-11-14T21:32:54,084 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting aeab510c920d424faba4f41bf6f73ae4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731619966711 2024-11-14T21:32:54,084 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting b85624253df64cc5ad9ed630c7850e25, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731619966711 2024-11-14T21:32:54,087 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting d711f2d085cd4a54a56f243a6c2bad65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731619968093 2024-11-14T21:32:54,087 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4925440b8b84e138e3fbe22f63fda64, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731619968093 2024-11-14T21:32:54,087 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting a392c672d5fa43bc99c0ba468de7441d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731619968829 2024-11-14T21:32:54,087 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c51e41421984860ae4f90890d080d37, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731619968829 2024-11-14T21:32:54,088 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dc60a577e6d490c9fc75ed7a6751e91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731619971108 2024-11-14T21:32:54,091 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96cedf9e6c5d43758fa5d2a3815faffe, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731619971108 2024-11-14T21:32:54,127 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:54,157 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 637fa2090a72f9209083e48908b752fe#B#compaction#150 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:54,158 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/534b1b6051684c828297718a1994060e is 50, key is test_row_0/B:col10/1731619971164/Put/seqid=0 2024-11-14T21:32:54,159 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411146e682dbe780a4175a2fcffab4697af36_637fa2090a72f9209083e48908b752fe store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:54,162 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411146e682dbe780a4175a2fcffab4697af36_637fa2090a72f9209083e48908b752fe, store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:54,163 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146e682dbe780a4175a2fcffab4697af36_637fa2090a72f9209083e48908b752fe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:54,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742002_1178 (size=12595) 2024-11-14T21:32:54,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742003_1179 (size=4469) 2024-11-14T21:32:54,244 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 637fa2090a72f9209083e48908b752fe#A#compaction#149 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:54,245 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/a0903ffb1980419cbc5c5fbbedbb58e8 is 175, key is test_row_0/A:col10/1731619971164/Put/seqid=0 2024-11-14T21:32:54,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742004_1180 (size=31549) 2024-11-14T21:32:54,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 637fa2090a72f9209083e48908b752fe 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T21:32:54,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=A 2024-11-14T21:32:54,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:54,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=B 2024-11-14T21:32:54,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:54,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=C 2024-11-14T21:32:54,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:54,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a9bc76beab6f4fafa3198c1f8d0ab2c9_637fa2090a72f9209083e48908b752fe is 50, key is test_row_0/A:col10/1731619973374/Put/seqid=0 2024-11-14T21:32:54,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742005_1181 (size=14794) 2024-11-14T21:32:54,623 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:54,628 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a9bc76beab6f4fafa3198c1f8d0ab2c9_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a9bc76beab6f4fafa3198c1f8d0ab2c9_637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:54,630 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/89dff48a1a5547d4b0f10311a2e6040d, store: [table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:54,630 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/534b1b6051684c828297718a1994060e as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/534b1b6051684c828297718a1994060e 2024-11-14T21:32:54,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/89dff48a1a5547d4b0f10311a2e6040d is 175, key is test_row_0/A:col10/1731619973374/Put/seqid=0 2024-11-14T21:32:54,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742006_1182 (size=39749) 2024-11-14T21:32:54,640 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/89dff48a1a5547d4b0f10311a2e6040d 2024-11-14T21:32:54,640 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 637fa2090a72f9209083e48908b752fe/B of 637fa2090a72f9209083e48908b752fe into 534b1b6051684c828297718a1994060e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:54,641 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:54,641 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., storeName=637fa2090a72f9209083e48908b752fe/B, priority=12, startTime=1731619974075; duration=0sec 2024-11-14T21:32:54,641 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:54,641 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 637fa2090a72f9209083e48908b752fe:B 2024-11-14T21:32:54,641 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:32:54,645 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:32:54,645 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 637fa2090a72f9209083e48908b752fe/C is initiating minor compaction (all files) 2024-11-14T21:32:54,645 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 637fa2090a72f9209083e48908b752fe/C in TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:54,645 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/bf92c06cd4504ef6a392eb62df1c7c08, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/ae20f27606a34cea88e7f20375ee8566, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/9d8e6571a3624ee3aa11980a8808e9c3, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/aa01ef10bf964156adb19ce62f7e1c1c] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp, totalSize=47.7 K 2024-11-14T21:32:54,646 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting bf92c06cd4504ef6a392eb62df1c7c08, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731619966711 2024-11-14T21:32:54,647 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting ae20f27606a34cea88e7f20375ee8566, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731619968093 2024-11-14T21:32:54,648 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d8e6571a3624ee3aa11980a8808e9c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731619968829 2024-11-14T21:32:54,648 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting aa01ef10bf964156adb19ce62f7e1c1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731619971108 2024-11-14T21:32:54,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/48ecec4ff74c42b4a075a03c47b5fe31 is 50, key is test_row_0/B:col10/1731619973374/Put/seqid=0 2024-11-14T21:32:54,679 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 637fa2090a72f9209083e48908b752fe#C#compaction#153 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:54,680 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/a91557c939f44e6bbb1958aa8e8e2820 is 50, key is test_row_0/C:col10/1731619971164/Put/seqid=0 2024-11-14T21:32:54,687 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/a0903ffb1980419cbc5c5fbbedbb58e8 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/a0903ffb1980419cbc5c5fbbedbb58e8 2024-11-14T21:32:54,698 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 637fa2090a72f9209083e48908b752fe/A of 637fa2090a72f9209083e48908b752fe into a0903ffb1980419cbc5c5fbbedbb58e8(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:54,698 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:54,698 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., storeName=637fa2090a72f9209083e48908b752fe/A, priority=12, startTime=1731619974075; duration=0sec 2024-11-14T21:32:54,698 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:54,698 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 637fa2090a72f9209083e48908b752fe:A 2024-11-14T21:32:54,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620034697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:54,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742007_1183 (size=12151) 2024-11-14T21:32:54,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742008_1184 (size=12595) 2024-11-14T21:32:54,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/48ecec4ff74c42b4a075a03c47b5fe31 2024-11-14T21:32:54,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/96bf6b9be1d041e3b023327fc91f2a3f is 50, key is test_row_0/C:col10/1731619973374/Put/seqid=0 2024-11-14T21:32:54,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742009_1185 (size=12151) 2024-11-14T21:32:54,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620034810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:54,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:54,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:54,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:55,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620035021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:55,121 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/a91557c939f44e6bbb1958aa8e8e2820 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/a91557c939f44e6bbb1958aa8e8e2820 2024-11-14T21:32:55,137 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 637fa2090a72f9209083e48908b752fe/C of 637fa2090a72f9209083e48908b752fe into a91557c939f44e6bbb1958aa8e8e2820(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:55,137 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:55,138 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., storeName=637fa2090a72f9209083e48908b752fe/C, priority=12, startTime=1731619974075; duration=0sec 2024-11-14T21:32:55,138 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:55,138 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 637fa2090a72f9209083e48908b752fe:C 2024-11-14T21:32:55,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/96bf6b9be1d041e3b023327fc91f2a3f 2024-11-14T21:32:55,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/89dff48a1a5547d4b0f10311a2e6040d as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/89dff48a1a5547d4b0f10311a2e6040d 2024-11-14T21:32:55,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/89dff48a1a5547d4b0f10311a2e6040d, entries=200, sequenceid=202, filesize=38.8 K 2024-11-14T21:32:55,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/48ecec4ff74c42b4a075a03c47b5fe31 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/48ecec4ff74c42b4a075a03c47b5fe31 2024-11-14T21:32:55,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/48ecec4ff74c42b4a075a03c47b5fe31, entries=150, sequenceid=202, filesize=11.9 K 2024-11-14T21:32:55,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/96bf6b9be1d041e3b023327fc91f2a3f as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/96bf6b9be1d041e3b023327fc91f2a3f 2024-11-14T21:32:55,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/96bf6b9be1d041e3b023327fc91f2a3f, entries=150, sequenceid=202, filesize=11.9 K 2024-11-14T21:32:55,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 637fa2090a72f9209083e48908b752fe in 674ms, sequenceid=202, compaction requested=false 2024-11-14T21:32:55,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:55,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 637fa2090a72f9209083e48908b752fe 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T21:32:55,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=A 2024-11-14T21:32:55,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:55,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=B 2024-11-14T21:32:55,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:55,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=C 2024-11-14T21:32:55,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:55,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146206923c6fd34e5a82729bee9d96badf_637fa2090a72f9209083e48908b752fe is 50, key is test_row_0/A:col10/1731619974677/Put/seqid=0 2024-11-14T21:32:55,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742010_1186 (size=14794) 2024-11-14T21:32:55,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620035438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:55,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:55,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:55,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:55,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620035550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:55,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:55,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:55,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:55,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620035799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:55,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:55,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:55,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:55,819 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:55,839 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146206923c6fd34e5a82729bee9d96badf_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146206923c6fd34e5a82729bee9d96badf_637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:55,841 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/5f5c0c23f6cd4ae78eb4e285097bb328, store: [table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:55,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/5f5c0c23f6cd4ae78eb4e285097bb328 is 175, key is test_row_0/A:col10/1731619974677/Put/seqid=0 2024-11-14T21:32:55,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742011_1187 (size=39749) 2024-11-14T21:32:56,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620036110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T21:32:56,191 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:32:56,191 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:32:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-14T21:32:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T21:32:56,195 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:32:56,195 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:32:56,196 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:32:56,283 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/5f5c0c23f6cd4ae78eb4e285097bb328 2024-11-14T21:32:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T21:32:56,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/86b23d4baec24d389b6fa9b7e96640c2 is 50, key is test_row_0/B:col10/1731619974677/Put/seqid=0 2024-11-14T21:32:56,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742012_1188 (size=12151) 2024-11-14T21:32:56,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/86b23d4baec24d389b6fa9b7e96640c2 2024-11-14T21:32:56,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T21:32:56,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:56,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:56,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:56,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:56,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:56,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:56,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/bc097eea7f7e42189d060f36c47863a4 is 50, key is test_row_0/C:col10/1731619974677/Put/seqid=0 2024-11-14T21:32:56,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742013_1189 (size=12151) 2024-11-14T21:32:56,375 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T21:32:56,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T21:32:56,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:56,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:56,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:56,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:56,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T21:32:56,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620036627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:56,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:56,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:56,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:56,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T21:32:56,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:56,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:56,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:56,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:56,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:56,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/bc097eea7f7e42189d060f36c47863a4 2024-11-14T21:32:56,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/5f5c0c23f6cd4ae78eb4e285097bb328 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/5f5c0c23f6cd4ae78eb4e285097bb328 2024-11-14T21:32:56,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/5f5c0c23f6cd4ae78eb4e285097bb328, entries=200, sequenceid=225, filesize=38.8 K 2024-11-14T21:32:56,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/86b23d4baec24d389b6fa9b7e96640c2 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/86b23d4baec24d389b6fa9b7e96640c2 2024-11-14T21:32:56,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/86b23d4baec24d389b6fa9b7e96640c2, entries=150, sequenceid=225, filesize=11.9 K 2024-11-14T21:32:56,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/bc097eea7f7e42189d060f36c47863a4 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/bc097eea7f7e42189d060f36c47863a4 2024-11-14T21:32:56,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/bc097eea7f7e42189d060f36c47863a4, entries=150, sequenceid=225, filesize=11.9 K 2024-11-14T21:32:56,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 637fa2090a72f9209083e48908b752fe in 1483ms, sequenceid=225, compaction requested=true 2024-11-14T21:32:56,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:56,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 637fa2090a72f9209083e48908b752fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:32:56,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:56,819 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:56,819 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:56,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 637fa2090a72f9209083e48908b752fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:32:56,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:56,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 637fa2090a72f9209083e48908b752fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:32:56,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:56,820 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:56,820 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:56,820 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 637fa2090a72f9209083e48908b752fe/A is initiating minor compaction (all files) 2024-11-14T21:32:56,820 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 637fa2090a72f9209083e48908b752fe/B is initiating minor compaction (all files) 2024-11-14T21:32:56,820 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 637fa2090a72f9209083e48908b752fe/A in TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:56,820 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 637fa2090a72f9209083e48908b752fe/B in TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T21:32:56,820 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/a0903ffb1980419cbc5c5fbbedbb58e8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/89dff48a1a5547d4b0f10311a2e6040d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/5f5c0c23f6cd4ae78eb4e285097bb328] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp, totalSize=108.4 K 2024-11-14T21:32:56,820 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/534b1b6051684c828297718a1994060e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/48ecec4ff74c42b4a075a03c47b5fe31, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/86b23d4baec24d389b6fa9b7e96640c2] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp, totalSize=36.0 K 2024-11-14T21:32:56,820 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:56,820 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. files: [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/a0903ffb1980419cbc5c5fbbedbb58e8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/89dff48a1a5547d4b0f10311a2e6040d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/5f5c0c23f6cd4ae78eb4e285097bb328] 2024-11-14T21:32:56,821 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 534b1b6051684c828297718a1994060e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731619971108 2024-11-14T21:32:56,821 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0903ffb1980419cbc5c5fbbedbb58e8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731619971108 2024-11-14T21:32:56,821 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 48ecec4ff74c42b4a075a03c47b5fe31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731619973374 2024-11-14T21:32:56,821 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89dff48a1a5547d4b0f10311a2e6040d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731619973354 2024-11-14T21:32:56,823 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 86b23d4baec24d389b6fa9b7e96640c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731619974677 2024-11-14T21:32:56,823 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f5c0c23f6cd4ae78eb4e285097bb328, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731619974671 2024-11-14T21:32:56,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T21:32:56,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:56,827 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 637fa2090a72f9209083e48908b752fe 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T21:32:56,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=A 2024-11-14T21:32:56,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:56,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=B 2024-11-14T21:32:56,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:56,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=C 2024-11-14T21:32:56,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:56,832 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 637fa2090a72f9209083e48908b752fe#B#compaction#158 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:56,833 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/2736e25b5c4d4cdebd367226207c9e44 is 50, key is test_row_0/B:col10/1731619974677/Put/seqid=0 2024-11-14T21:32:56,839 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:56,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148de9105c1991488fbf1ccbd23ed82059_637fa2090a72f9209083e48908b752fe is 50, key is test_row_0/A:col10/1731619975432/Put/seqid=0 2024-11-14T21:32:56,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742014_1190 (size=12697) 2024-11-14T21:32:56,859 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111414230a22344345e587eec821757e8616_637fa2090a72f9209083e48908b752fe store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:56,861 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111414230a22344345e587eec821757e8616_637fa2090a72f9209083e48908b752fe, store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:56,862 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111414230a22344345e587eec821757e8616_637fa2090a72f9209083e48908b752fe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:56,869 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/2736e25b5c4d4cdebd367226207c9e44 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/2736e25b5c4d4cdebd367226207c9e44 2024-11-14T21:32:56,877 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 637fa2090a72f9209083e48908b752fe/B of 637fa2090a72f9209083e48908b752fe into 2736e25b5c4d4cdebd367226207c9e44(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:56,877 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:56,877 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., storeName=637fa2090a72f9209083e48908b752fe/B, priority=13, startTime=1731619976819; duration=0sec 2024-11-14T21:32:56,877 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:56,877 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 637fa2090a72f9209083e48908b752fe:B 2024-11-14T21:32:56,877 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:56,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742015_1191 (size=12304) 2024-11-14T21:32:56,879 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:56,880 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 637fa2090a72f9209083e48908b752fe/C is initiating minor compaction (all files) 2024-11-14T21:32:56,880 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 637fa2090a72f9209083e48908b752fe/C in TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:56,880 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/a91557c939f44e6bbb1958aa8e8e2820, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/96bf6b9be1d041e3b023327fc91f2a3f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/bc097eea7f7e42189d060f36c47863a4] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp, totalSize=36.0 K 2024-11-14T21:32:56,884 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting a91557c939f44e6bbb1958aa8e8e2820, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731619971108 2024-11-14T21:32:56,884 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 96bf6b9be1d041e3b023327fc91f2a3f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731619973374 2024-11-14T21:32:56,885 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting bc097eea7f7e42189d060f36c47863a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731619974677 2024-11-14T21:32:56,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:56,909 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 637fa2090a72f9209083e48908b752fe#C#compaction#161 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:56,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742016_1192 (size=4469) 2024-11-14T21:32:56,910 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/bd8f17c4b64f4570b4ed1dc0ca4046c0 is 50, key is test_row_0/C:col10/1731619974677/Put/seqid=0 2024-11-14T21:32:56,911 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148de9105c1991488fbf1ccbd23ed82059_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148de9105c1991488fbf1ccbd23ed82059_637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:56,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/ec18596e19ca4a41b3b7c4340c7bfb56, store: [table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:56,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/ec18596e19ca4a41b3b7c4340c7bfb56 is 175, key is test_row_0/A:col10/1731619975432/Put/seqid=0 2024-11-14T21:32:56,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742017_1193 (size=12697) 2024-11-14T21:32:56,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742018_1194 (size=31105) 2024-11-14T21:32:56,923 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/ec18596e19ca4a41b3b7c4340c7bfb56 2024-11-14T21:32:56,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/214ec4a239b745eb96ba46289a0cdf00 is 50, key is test_row_0/B:col10/1731619975432/Put/seqid=0 2024-11-14T21:32:56,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742019_1195 (size=12151) 2024-11-14T21:32:57,313 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 637fa2090a72f9209083e48908b752fe#A#compaction#159 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:57,314 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/020a5c37f4ec49c985950433725a3d4b is 175, key is test_row_0/A:col10/1731619974677/Put/seqid=0 2024-11-14T21:32:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T21:32:57,340 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/bd8f17c4b64f4570b4ed1dc0ca4046c0 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/bd8f17c4b64f4570b4ed1dc0ca4046c0 2024-11-14T21:32:57,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742020_1196 (size=31651) 2024-11-14T21:32:57,346 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/214ec4a239b745eb96ba46289a0cdf00 2024-11-14T21:32:57,348 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 637fa2090a72f9209083e48908b752fe/C of 637fa2090a72f9209083e48908b752fe into bd8f17c4b64f4570b4ed1dc0ca4046c0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:57,348 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:57,348 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., storeName=637fa2090a72f9209083e48908b752fe/C, priority=13, startTime=1731619976819; duration=0sec 2024-11-14T21:32:57,348 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:57,348 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 637fa2090a72f9209083e48908b752fe:C 2024-11-14T21:32:57,356 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/020a5c37f4ec49c985950433725a3d4b as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/020a5c37f4ec49c985950433725a3d4b 2024-11-14T21:32:57,365 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 637fa2090a72f9209083e48908b752fe/A of 637fa2090a72f9209083e48908b752fe into 020a5c37f4ec49c985950433725a3d4b(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:57,365 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:57,366 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., storeName=637fa2090a72f9209083e48908b752fe/A, priority=13, startTime=1731619976818; duration=0sec 2024-11-14T21:32:57,366 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:57,366 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 637fa2090a72f9209083e48908b752fe:A 2024-11-14T21:32:57,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/a6c872f2a6854f7bb14f92ad3d317646 is 50, key is test_row_0/C:col10/1731619975432/Put/seqid=0 2024-11-14T21:32:57,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742021_1197 (size=12151) 2024-11-14T21:32:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:57,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:57,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:57,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620037747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:57,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:57,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:57,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:57,828 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/a6c872f2a6854f7bb14f92ad3d317646 2024-11-14T21:32:57,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/ec18596e19ca4a41b3b7c4340c7bfb56 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/ec18596e19ca4a41b3b7c4340c7bfb56 2024-11-14T21:32:57,853 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/ec18596e19ca4a41b3b7c4340c7bfb56, entries=150, sequenceid=237, filesize=30.4 K 2024-11-14T21:32:57,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/214ec4a239b745eb96ba46289a0cdf00 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/214ec4a239b745eb96ba46289a0cdf00 2024-11-14T21:32:57,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:57,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620037860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:57,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:57,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:57,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:57,911 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/214ec4a239b745eb96ba46289a0cdf00, entries=150, sequenceid=237, filesize=11.9 K 2024-11-14T21:32:57,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/a6c872f2a6854f7bb14f92ad3d317646 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/a6c872f2a6854f7bb14f92ad3d317646 2024-11-14T21:32:57,947 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/a6c872f2a6854f7bb14f92ad3d317646, entries=150, sequenceid=237, filesize=11.9 K 2024-11-14T21:32:57,948 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 637fa2090a72f9209083e48908b752fe in 1121ms, sequenceid=237, compaction requested=false 2024-11-14T21:32:57,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:57,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:57,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-14T21:32:57,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-14T21:32:57,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-14T21:32:57,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7780 sec 2024-11-14T21:32:57,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.7990 sec 2024-11-14T21:32:58,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:58,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 637fa2090a72f9209083e48908b752fe 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-14T21:32:58,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=A 2024-11-14T21:32:58,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:32:58,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=B 2024-11-14T21:32:58,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:32:58,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=C 2024-11-14T21:32:58,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:32:58,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bc27d8aa35234cce8dc4c11403adaed8_637fa2090a72f9209083e48908b752fe is 50, key is test_row_0/A:col10/1731619978082/Put/seqid=0 2024-11-14T21:32:58,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742022_1198 (size=14944) 2024-11-14T21:32:58,133 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:58,138 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bc27d8aa35234cce8dc4c11403adaed8_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bc27d8aa35234cce8dc4c11403adaed8_637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:58,139 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/12e0824f72d2474da39ba59cf69a3a90, store: [table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:58,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:58,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620038133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:58,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/12e0824f72d2474da39ba59cf69a3a90 is 175, key is test_row_0/A:col10/1731619978082/Put/seqid=0 2024-11-14T21:32:58,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:58,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:58,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:58,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742023_1199 (size=39899) 2024-11-14T21:32:58,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:58,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620038255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T21:32:58,341 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:32:58,342 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:32:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-14T21:32:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T21:32:58,352 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:32:58,353 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:32:58,353 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:32:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T21:32:58,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:58,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620038472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:58,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:58,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:58,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:58,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T21:32:58,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:58,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:58,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:58,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:58,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:58,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:58,566 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=261, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/12e0824f72d2474da39ba59cf69a3a90 2024-11-14T21:32:58,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/1609cb128a3a45849cbd19b0a5c4daaf is 50, key is test_row_0/B:col10/1731619978082/Put/seqid=0 2024-11-14T21:32:58,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742024_1200 (size=12251) 2024-11-14T21:32:58,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/1609cb128a3a45849cbd19b0a5c4daaf 2024-11-14T21:32:58,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/72105dd4cd3e4743a84e55caddb53703 is 50, key is test_row_0/C:col10/1731619978082/Put/seqid=0 2024-11-14T21:32:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T21:32:58,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T21:32:58,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:58,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:58,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:58,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:58,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:58,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:58,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742025_1201 (size=12251) 2024-11-14T21:32:58,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:32:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53730 deadline: 1731620038791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:32:58,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:58,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:32:58,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:32:58,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T21:32:58,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:58,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:58,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:58,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:58,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:58,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:58,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T21:32:58,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T21:32:58,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:58,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:58,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:58,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:59,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:59,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:32:59,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/72105dd4cd3e4743a84e55caddb53703 2024-11-14T21:32:59,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/12e0824f72d2474da39ba59cf69a3a90 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/12e0824f72d2474da39ba59cf69a3a90 2024-11-14T21:32:59,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/12e0824f72d2474da39ba59cf69a3a90, entries=200, sequenceid=261, filesize=39.0 K 2024-11-14T21:32:59,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/1609cb128a3a45849cbd19b0a5c4daaf as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/1609cb128a3a45849cbd19b0a5c4daaf 2024-11-14T21:32:59,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/1609cb128a3a45849cbd19b0a5c4daaf, entries=150, sequenceid=261, filesize=12.0 K 2024-11-14T21:32:59,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/72105dd4cd3e4743a84e55caddb53703 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/72105dd4cd3e4743a84e55caddb53703 2024-11-14T21:32:59,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/72105dd4cd3e4743a84e55caddb53703, entries=150, sequenceid=261, filesize=12.0 K 2024-11-14T21:32:59,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 637fa2090a72f9209083e48908b752fe in 1069ms, sequenceid=261, compaction requested=true 2024-11-14T21:32:59,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:59,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 637fa2090a72f9209083e48908b752fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:32:59,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:59,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 637fa2090a72f9209083e48908b752fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:32:59,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:32:59,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 637fa2090a72f9209083e48908b752fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:32:59,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T21:32:59,154 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:59,154 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:59,156 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102655 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:59,156 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 637fa2090a72f9209083e48908b752fe/A is initiating minor compaction (all files) 2024-11-14T21:32:59,156 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 637fa2090a72f9209083e48908b752fe/A in TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:59,157 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/020a5c37f4ec49c985950433725a3d4b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/ec18596e19ca4a41b3b7c4340c7bfb56, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/12e0824f72d2474da39ba59cf69a3a90] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp, totalSize=100.2 K 2024-11-14T21:32:59,157 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:59,157 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. files: [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/020a5c37f4ec49c985950433725a3d4b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/ec18596e19ca4a41b3b7c4340c7bfb56, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/12e0824f72d2474da39ba59cf69a3a90] 2024-11-14T21:32:59,157 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:59,157 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 637fa2090a72f9209083e48908b752fe/C is initiating minor compaction (all files) 2024-11-14T21:32:59,157 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 637fa2090a72f9209083e48908b752fe/C in TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:59,157 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/bd8f17c4b64f4570b4ed1dc0ca4046c0, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/a6c872f2a6854f7bb14f92ad3d317646, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/72105dd4cd3e4743a84e55caddb53703] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp, totalSize=36.2 K 2024-11-14T21:32:59,158 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting bd8f17c4b64f4570b4ed1dc0ca4046c0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731619974677 2024-11-14T21:32:59,158 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 020a5c37f4ec49c985950433725a3d4b, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731619974677 2024-11-14T21:32:59,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T21:32:59,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:59,159 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting a6c872f2a6854f7bb14f92ad3d317646, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731619975417 2024-11-14T21:32:59,159 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 637fa2090a72f9209083e48908b752fe 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T21:32:59,159 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec18596e19ca4a41b3b7c4340c7bfb56, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731619975417 2024-11-14T21:32:59,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=A 2024-11-14T21:32:59,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:59,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=B 2024-11-14T21:32:59,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:59,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=C 2024-11-14T21:32:59,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:32:59,159 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 72105dd4cd3e4743a84e55caddb53703, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731619977716 2024-11-14T21:32:59,160 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12e0824f72d2474da39ba59cf69a3a90, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731619977716 2024-11-14T21:32:59,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111477aef402d9a84f169c7225eff8001d1b_637fa2090a72f9209083e48908b752fe is 50, key is test_row_0/A:col10/1731619978131/Put/seqid=0 2024-11-14T21:32:59,205 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 637fa2090a72f9209083e48908b752fe#C#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:59,206 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/f001e70616464042a1b75d43a5f74ccb is 50, key is test_row_0/C:col10/1731619978082/Put/seqid=0 2024-11-14T21:32:59,218 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:59,246 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114a1b7779737d6475eade7d79f2085174c_637fa2090a72f9209083e48908b752fe store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:59,250 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114a1b7779737d6475eade7d79f2085174c_637fa2090a72f9209083e48908b752fe, store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:59,251 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a1b7779737d6475eade7d79f2085174c_637fa2090a72f9209083e48908b752fe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:59,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742026_1202 (size=12454) 2024-11-14T21:32:59,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:32:59,286 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111477aef402d9a84f169c7225eff8001d1b_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111477aef402d9a84f169c7225eff8001d1b_637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:59,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/a0d1c73988dc4c8baa33e2d282cee1dd, store: [table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:32:59,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/a0d1c73988dc4c8baa33e2d282cee1dd is 175, key is test_row_0/A:col10/1731619978131/Put/seqid=0 2024-11-14T21:32:59,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742027_1203 (size=12899) 2024-11-14T21:32:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 637fa2090a72f9209083e48908b752fe 2024-11-14T21:32:59,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742028_1204 (size=4469) 2024-11-14T21:32:59,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. as already flushing 2024-11-14T21:32:59,318 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 637fa2090a72f9209083e48908b752fe#A#compaction#169 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:59,319 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/78a4c3ed292541f6abcb76384b0ef835 is 175, key is test_row_0/A:col10/1731619978082/Put/seqid=0 2024-11-14T21:32:59,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742029_1205 (size=31255) 2024-11-14T21:32:59,356 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/a0d1c73988dc4c8baa33e2d282cee1dd 2024-11-14T21:32:59,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742030_1206 (size=31853) 2024-11-14T21:32:59,382 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/78a4c3ed292541f6abcb76384b0ef835 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/78a4c3ed292541f6abcb76384b0ef835 2024-11-14T21:32:59,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/17c90e081ee6417d9fb1582c9ac04523 is 50, key is test_row_0/B:col10/1731619978131/Put/seqid=0 2024-11-14T21:32:59,411 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 637fa2090a72f9209083e48908b752fe/A of 637fa2090a72f9209083e48908b752fe into 78a4c3ed292541f6abcb76384b0ef835(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:59,411 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:59,412 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., storeName=637fa2090a72f9209083e48908b752fe/A, priority=13, startTime=1731619979153; duration=0sec 2024-11-14T21:32:59,412 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:32:59,412 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 637fa2090a72f9209083e48908b752fe:A 2024-11-14T21:32:59,413 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:32:59,419 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:32:59,419 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 637fa2090a72f9209083e48908b752fe/B is initiating minor compaction (all files) 2024-11-14T21:32:59,420 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 637fa2090a72f9209083e48908b752fe/B in TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:59,420 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/2736e25b5c4d4cdebd367226207c9e44, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/214ec4a239b745eb96ba46289a0cdf00, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/1609cb128a3a45849cbd19b0a5c4daaf] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp, totalSize=36.2 K 2024-11-14T21:32:59,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742031_1207 (size=12301) 2024-11-14T21:32:59,429 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2736e25b5c4d4cdebd367226207c9e44, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731619974677 2024-11-14T21:32:59,430 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/17c90e081ee6417d9fb1582c9ac04523 2024-11-14T21:32:59,434 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 214ec4a239b745eb96ba46289a0cdf00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731619975417 2024-11-14T21:32:59,439 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1609cb128a3a45849cbd19b0a5c4daaf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731619977716 2024-11-14T21:32:59,478 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 637fa2090a72f9209083e48908b752fe#B#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:32:59,478 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/40bb7cd5d3e74ab48d684c1440bd601e is 50, key is test_row_0/B:col10/1731619978082/Put/seqid=0 2024-11-14T21:32:59,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/d0ed8dd21b3e4154a4e121aac5a83ca8 is 50, key is test_row_0/C:col10/1731619978131/Put/seqid=0 2024-11-14T21:32:59,498 INFO [Thread-652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-652. 2024-11-14T21:32:59,498 DEBUG [Thread-652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:32:59,498 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:59,498 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:59,499 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:32:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T21:32:59,504 INFO [Thread-646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-646. 2024-11-14T21:32:59,505 DEBUG [Thread-646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:32:59,505 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:59,505 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:59,505 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:32:59,508 INFO [Thread-641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-641. 2024-11-14T21:32:59,508 DEBUG [Thread-641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:32:59,508 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:59,508 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:59,510 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:32:59,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731619979455, max timestamp=1731619979506Num uniques 859; 2024-11-14T21:32:59,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:32:59,515 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-11-14T21:32:59,515 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:32:59,515 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:59,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731619979455, max timestamp=1731619979506Num uniques 859; 2024-11-14T21:32:59,515 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:59,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:32:59,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:32:59,517 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-11-14T21:32:59,517 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:32:59,517 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:59,517 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:32:59,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742032_1208 (size=12899) 2024-11-14T21:32:59,517 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:32:59,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742033_1209 (size=12301) 2024-11-14T21:32:59,546 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/40bb7cd5d3e74ab48d684c1440bd601e as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/40bb7cd5d3e74ab48d684c1440bd601e 2024-11-14T21:32:59,563 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 637fa2090a72f9209083e48908b752fe/B of 637fa2090a72f9209083e48908b752fe into 40bb7cd5d3e74ab48d684c1440bd601e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:59,563 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:59,563 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., storeName=637fa2090a72f9209083e48908b752fe/B, priority=13, startTime=1731619979154; duration=0sec 2024-11-14T21:32:59,563 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:59,563 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 637fa2090a72f9209083e48908b752fe:B 2024-11-14T21:32:59,711 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/f001e70616464042a1b75d43a5f74ccb as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/f001e70616464042a1b75d43a5f74ccb 2024-11-14T21:32:59,717 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 637fa2090a72f9209083e48908b752fe/C of 637fa2090a72f9209083e48908b752fe into f001e70616464042a1b75d43a5f74ccb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:32:59,717 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:59,717 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., storeName=637fa2090a72f9209083e48908b752fe/C, priority=13, startTime=1731619979154; duration=0sec 2024-11-14T21:32:59,717 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:32:59,717 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 637fa2090a72f9209083e48908b752fe:C 2024-11-14T21:32:59,942 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/d0ed8dd21b3e4154a4e121aac5a83ca8 2024-11-14T21:32:59,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/a0d1c73988dc4c8baa33e2d282cee1dd as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/a0d1c73988dc4c8baa33e2d282cee1dd 2024-11-14T21:32:59,950 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/a0d1c73988dc4c8baa33e2d282cee1dd, entries=150, sequenceid=272, filesize=30.5 K 2024-11-14T21:32:59,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/17c90e081ee6417d9fb1582c9ac04523 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/17c90e081ee6417d9fb1582c9ac04523 2024-11-14T21:32:59,954 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/17c90e081ee6417d9fb1582c9ac04523, entries=150, sequenceid=272, filesize=12.0 K 2024-11-14T21:32:59,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/d0ed8dd21b3e4154a4e121aac5a83ca8 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/d0ed8dd21b3e4154a4e121aac5a83ca8 2024-11-14T21:32:59,958 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/d0ed8dd21b3e4154a4e121aac5a83ca8, entries=150, sequenceid=272, filesize=12.0 K 2024-11-14T21:32:59,959 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=67.09 KB/68700 for 637fa2090a72f9209083e48908b752fe in 800ms, sequenceid=272, compaction requested=false 2024-11-14T21:32:59,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:32:59,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:32:59,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-14T21:32:59,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-14T21:32:59,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-14T21:32:59,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6070 sec 2024-11-14T21:32:59,964 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.6210 sec 2024-11-14T21:33:00,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T21:33:00,511 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:33:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:01,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 637fa2090a72f9209083e48908b752fe 3/3 column families, dataSize=73.80 KB heapSize=204.41 KB 2024-11-14T21:33:01,538 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-14T21:33:01,538 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:33:01,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=A 2024-11-14T21:33:01,538 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:01,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:01,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=B 2024-11-14T21:33:01,538 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:01,538 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:33:01,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:01,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=C 2024-11-14T21:33:01,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:01,584 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-11-14T21:33:01,584 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:33:01,584 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:01,585 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:01,586 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:33:01,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111494a6489d011c4a7bb48e25acd8f041c7_637fa2090a72f9209083e48908b752fe is 50, key is test_row_0/A:col10/1731619979471/Put/seqid=0 2024-11-14T21:33:01,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742034_1210 (size=12454) 2024-11-14T21:33:02,011 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:02,016 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111494a6489d011c4a7bb48e25acd8f041c7_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111494a6489d011c4a7bb48e25acd8f041c7_637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:02,017 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/8814e09fe9364057adab19c7c572fe29, store: [table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:33:02,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/8814e09fe9364057adab19c7c572fe29 is 175, key is test_row_0/A:col10/1731619979471/Put/seqid=0 2024-11-14T21:33:02,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742035_1211 (size=31255) 2024-11-14T21:33:02,392 INFO [Thread-643 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-643. 2024-11-14T21:33:02,392 DEBUG [Thread-643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:33:02,392 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:02,392 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:02,392 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:33:02,413 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-11-14T21:33:02,413 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:33:02,413 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:02,413 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:02,413 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:33:02,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T21:33:02,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-14T21:33:02,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-14T21:33:02,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-14T21:33:02,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 162 2024-11-14T21:33:02,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-14T21:33:02,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T21:33:02,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2082 2024-11-14T21:33:02,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2101 2024-11-14T21:33:02,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T21:33:02,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1000 2024-11-14T21:33:02,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2996 rows 2024-11-14T21:33:02,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 983 2024-11-14T21:33:02,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2948 rows 2024-11-14T21:33:02,414 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T21:33:02,414 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T21:33:02,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:02,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:02,415 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:33:02,416 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T21:33:02,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T21:33:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T21:33:02,422 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731619982421"}]},"ts":"1731619982421"} 2024-11-14T21:33:02,423 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/8814e09fe9364057adab19c7c572fe29 2024-11-14T21:33:02,424 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T21:33:02,424 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T21:33:02,425 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T21:33:02,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=637fa2090a72f9209083e48908b752fe, UNASSIGN}] 2024-11-14T21:33:02,428 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=637fa2090a72f9209083e48908b752fe, UNASSIGN 2024-11-14T21:33:02,429 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=48 updating hbase:meta row=637fa2090a72f9209083e48908b752fe, regionState=CLOSING, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:02,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=637fa2090a72f9209083e48908b752fe, UNASSIGN because future has completed 2024-11-14T21:33:02,431 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T21:33:02,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; CloseRegionProcedure 637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560}] 2024-11-14T21:33:02,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/eaa947a186bb45aba359ca24dc5f34f2 is 50, key is test_row_0/B:col10/1731619979471/Put/seqid=0 2024-11-14T21:33:02,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742036_1212 (size=12301) 2024-11-14T21:33:02,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/eaa947a186bb45aba359ca24dc5f34f2 2024-11-14T21:33:02,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/03c8cb75a4ac4e8288778921b0d0dea5 is 50, key is test_row_0/C:col10/1731619979471/Put/seqid=0 2024-11-14T21:33:02,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742037_1213 (size=12301) 2024-11-14T21:33:02,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T21:33:02,584 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] handler.UnassignRegionHandler(122): Close 637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:02,584 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T21:33:02,585 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1722): Closing 637fa2090a72f9209083e48908b752fe, disabling compactions & flushes 2024-11-14T21:33:02,585 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:33:02,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T21:33:02,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/03c8cb75a4ac4e8288778921b0d0dea5 2024-11-14T21:33:02,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/8814e09fe9364057adab19c7c572fe29 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/8814e09fe9364057adab19c7c572fe29 2024-11-14T21:33:02,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/8814e09fe9364057adab19c7c572fe29, entries=150, sequenceid=297, filesize=30.5 K 2024-11-14T21:33:02,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/eaa947a186bb45aba359ca24dc5f34f2 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/eaa947a186bb45aba359ca24dc5f34f2 2024-11-14T21:33:02,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/eaa947a186bb45aba359ca24dc5f34f2, entries=150, sequenceid=297, filesize=12.0 K 2024-11-14T21:33:02,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/03c8cb75a4ac4e8288778921b0d0dea5 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/03c8cb75a4ac4e8288778921b0d0dea5 2024-11-14T21:33:02,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/03c8cb75a4ac4e8288778921b0d0dea5, entries=150, sequenceid=297, filesize=12.0 K 2024-11-14T21:33:02,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~204.36 KB/209264, currentSize=20.13 KB/20610 for 637fa2090a72f9209083e48908b752fe in 1348ms, sequenceid=297, compaction requested=true 2024-11-14T21:33:02,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 637fa2090a72f9209083e48908b752fe: 2024-11-14T21:33:02,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 637fa2090a72f9209083e48908b752fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:33:02,885 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:33:02,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:02,885 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:33:02,885 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. because compaction request was cancelled 2024-11-14T21:33:02,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 637fa2090a72f9209083e48908b752fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:33:02,885 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. after waiting 0 ms 2024-11-14T21:33:02,885 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 637fa2090a72f9209083e48908b752fe:A 2024-11-14T21:33:02,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:02,885 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:33:02,885 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. because compaction request was cancelled 2024-11-14T21:33:02,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 637fa2090a72f9209083e48908b752fe:C, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:33:02,885 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. because compaction request was cancelled 2024-11-14T21:33:02,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:02,885 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 637fa2090a72f9209083e48908b752fe:B 2024-11-14T21:33:02,885 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 637fa2090a72f9209083e48908b752fe:C 2024-11-14T21:33:02,885 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(2902): Flushing 637fa2090a72f9209083e48908b752fe 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-14T21:33:02,886 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=A 2024-11-14T21:33:02,886 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:02,886 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=B 2024-11-14T21:33:02,886 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:02,886 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 637fa2090a72f9209083e48908b752fe, store=C 2024-11-14T21:33:02,886 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:02,890 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141f83978d565e4d88b57273008ce6337c_637fa2090a72f9209083e48908b752fe is 50, key is test_row_0/A:col10/1731619982411/Put/seqid=0 2024-11-14T21:33:02,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742038_1214 (size=12454) 2024-11-14T21:33:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T21:33:03,294 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:03,299 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141f83978d565e4d88b57273008ce6337c_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141f83978d565e4d88b57273008ce6337c_637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:03,300 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/21b133e3571f4fc8a99a6c5a6fba3f5c, store: [table=TestAcidGuarantees family=A region=637fa2090a72f9209083e48908b752fe] 2024-11-14T21:33:03,301 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/21b133e3571f4fc8a99a6c5a6fba3f5c is 175, key is test_row_0/A:col10/1731619982411/Put/seqid=0 2024-11-14T21:33:03,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742039_1215 (size=31255) 2024-11-14T21:33:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T21:33:03,706 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/21b133e3571f4fc8a99a6c5a6fba3f5c 2024-11-14T21:33:03,713 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/fd348512253f427f8bee9ba70621c65c is 50, key is test_row_0/B:col10/1731619982411/Put/seqid=0 2024-11-14T21:33:03,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742040_1216 (size=12301) 2024-11-14T21:33:04,117 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/fd348512253f427f8bee9ba70621c65c 2024-11-14T21:33:04,127 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/ade086ec1a2c4b0bb299007c071ad353 is 50, key is test_row_0/C:col10/1731619982411/Put/seqid=0 2024-11-14T21:33:04,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742041_1217 (size=12301) 2024-11-14T21:33:04,535 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/ade086ec1a2c4b0bb299007c071ad353 2024-11-14T21:33:04,546 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/A/21b133e3571f4fc8a99a6c5a6fba3f5c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/21b133e3571f4fc8a99a6c5a6fba3f5c 2024-11-14T21:33:04,551 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/21b133e3571f4fc8a99a6c5a6fba3f5c, entries=150, sequenceid=303, filesize=30.5 K 2024-11-14T21:33:04,552 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/B/fd348512253f427f8bee9ba70621c65c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/fd348512253f427f8bee9ba70621c65c 2024-11-14T21:33:04,563 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/fd348512253f427f8bee9ba70621c65c, entries=150, sequenceid=303, filesize=12.0 K 2024-11-14T21:33:04,570 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/.tmp/C/ade086ec1a2c4b0bb299007c071ad353 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/ade086ec1a2c4b0bb299007c071ad353 2024-11-14T21:33:04,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T21:33:04,604 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/ade086ec1a2c4b0bb299007c071ad353, entries=150, sequenceid=303, filesize=12.0 K 2024-11-14T21:33:04,607 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 637fa2090a72f9209083e48908b752fe in 1721ms, sequenceid=303, compaction requested=true 2024-11-14T21:33:04,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/4290bdf939514c79ade90297625292ea, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/195e06f53ee74a938e70c18704c82896, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/4710fb94d3f347ddb7b95fe9bc41313d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/c61e45cc046c47a89f9ca95cfff2309f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/e843be2386274003a341a838deb1a9bb, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/161963451ed04eb2a5b40e3e72bd3685, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/057caa89e18d41069b3ea7db14293f23, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/7360d333eebb4b3fa609ac5c8649c123, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/b85624253df64cc5ad9ed630c7850e25, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/56a2d7c03fa4463bb03fea8b2cf07f2e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/c4925440b8b84e138e3fbe22f63fda64, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/9c51e41421984860ae4f90890d080d37, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/96cedf9e6c5d43758fa5d2a3815faffe, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/a0903ffb1980419cbc5c5fbbedbb58e8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/89dff48a1a5547d4b0f10311a2e6040d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/5f5c0c23f6cd4ae78eb4e285097bb328, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/020a5c37f4ec49c985950433725a3d4b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/ec18596e19ca4a41b3b7c4340c7bfb56, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/12e0824f72d2474da39ba59cf69a3a90] to archive 2024-11-14T21:33:04,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:33:04,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/4290bdf939514c79ade90297625292ea to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/4290bdf939514c79ade90297625292ea 2024-11-14T21:33:04,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/195e06f53ee74a938e70c18704c82896 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/195e06f53ee74a938e70c18704c82896 2024-11-14T21:33:04,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/4710fb94d3f347ddb7b95fe9bc41313d to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/4710fb94d3f347ddb7b95fe9bc41313d 2024-11-14T21:33:04,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/c61e45cc046c47a89f9ca95cfff2309f to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/c61e45cc046c47a89f9ca95cfff2309f 2024-11-14T21:33:04,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/e843be2386274003a341a838deb1a9bb to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/e843be2386274003a341a838deb1a9bb 2024-11-14T21:33:04,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/161963451ed04eb2a5b40e3e72bd3685 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/161963451ed04eb2a5b40e3e72bd3685 2024-11-14T21:33:04,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/057caa89e18d41069b3ea7db14293f23 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/057caa89e18d41069b3ea7db14293f23 2024-11-14T21:33:04,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/7360d333eebb4b3fa609ac5c8649c123 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/7360d333eebb4b3fa609ac5c8649c123 2024-11-14T21:33:04,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/b85624253df64cc5ad9ed630c7850e25 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/b85624253df64cc5ad9ed630c7850e25 2024-11-14T21:33:04,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/56a2d7c03fa4463bb03fea8b2cf07f2e to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/56a2d7c03fa4463bb03fea8b2cf07f2e 2024-11-14T21:33:04,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/c4925440b8b84e138e3fbe22f63fda64 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/c4925440b8b84e138e3fbe22f63fda64 2024-11-14T21:33:04,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/9c51e41421984860ae4f90890d080d37 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/9c51e41421984860ae4f90890d080d37 2024-11-14T21:33:04,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/96cedf9e6c5d43758fa5d2a3815faffe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/96cedf9e6c5d43758fa5d2a3815faffe 2024-11-14T21:33:04,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/a0903ffb1980419cbc5c5fbbedbb58e8 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/a0903ffb1980419cbc5c5fbbedbb58e8 2024-11-14T21:33:04,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/89dff48a1a5547d4b0f10311a2e6040d to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/89dff48a1a5547d4b0f10311a2e6040d 2024-11-14T21:33:04,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/5f5c0c23f6cd4ae78eb4e285097bb328 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/5f5c0c23f6cd4ae78eb4e285097bb328 2024-11-14T21:33:04,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/020a5c37f4ec49c985950433725a3d4b to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/020a5c37f4ec49c985950433725a3d4b 2024-11-14T21:33:04,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/ec18596e19ca4a41b3b7c4340c7bfb56 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/ec18596e19ca4a41b3b7c4340c7bfb56 2024-11-14T21:33:04,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/12e0824f72d2474da39ba59cf69a3a90 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/12e0824f72d2474da39ba59cf69a3a90 2024-11-14T21:33:04,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/796aa3be47e145f18db4bc5950d8b383, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/382304556f41409ebd3b404ee1181144, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/e07da9d63e3d4b1f8d3cbdaf4a44b879, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/d6ecd06f861a4121b3db2c15752bc0d3, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/8007a6c19753456c99a15b588cb7cc0b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/2c598bfcd8a84679852b893f50bc3204, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/ab34b797974c43d5b6eb7e6f33e61d40, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/1d726febb1e1436a8f02cf33f7bbaba8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/aeab510c920d424faba4f41bf6f73ae4, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/1f2e848cb1154555b51fa82b64f2c243, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/d711f2d085cd4a54a56f243a6c2bad65, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/a392c672d5fa43bc99c0ba468de7441d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/534b1b6051684c828297718a1994060e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/2dc60a577e6d490c9fc75ed7a6751e91, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/48ecec4ff74c42b4a075a03c47b5fe31, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/2736e25b5c4d4cdebd367226207c9e44, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/86b23d4baec24d389b6fa9b7e96640c2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/214ec4a239b745eb96ba46289a0cdf00, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/1609cb128a3a45849cbd19b0a5c4daaf] to archive 2024-11-14T21:33:04,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:33:04,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/796aa3be47e145f18db4bc5950d8b383 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/796aa3be47e145f18db4bc5950d8b383 2024-11-14T21:33:04,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/382304556f41409ebd3b404ee1181144 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/382304556f41409ebd3b404ee1181144 2024-11-14T21:33:04,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/e07da9d63e3d4b1f8d3cbdaf4a44b879 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/e07da9d63e3d4b1f8d3cbdaf4a44b879 2024-11-14T21:33:04,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/d6ecd06f861a4121b3db2c15752bc0d3 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/d6ecd06f861a4121b3db2c15752bc0d3 2024-11-14T21:33:04,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/8007a6c19753456c99a15b588cb7cc0b to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/8007a6c19753456c99a15b588cb7cc0b 2024-11-14T21:33:04,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/2c598bfcd8a84679852b893f50bc3204 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/2c598bfcd8a84679852b893f50bc3204 2024-11-14T21:33:04,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/ab34b797974c43d5b6eb7e6f33e61d40 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/ab34b797974c43d5b6eb7e6f33e61d40 2024-11-14T21:33:04,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/1d726febb1e1436a8f02cf33f7bbaba8 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/1d726febb1e1436a8f02cf33f7bbaba8 2024-11-14T21:33:04,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/aeab510c920d424faba4f41bf6f73ae4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/aeab510c920d424faba4f41bf6f73ae4 2024-11-14T21:33:04,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/1f2e848cb1154555b51fa82b64f2c243 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/1f2e848cb1154555b51fa82b64f2c243 2024-11-14T21:33:04,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/d711f2d085cd4a54a56f243a6c2bad65 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/d711f2d085cd4a54a56f243a6c2bad65 2024-11-14T21:33:04,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/a392c672d5fa43bc99c0ba468de7441d to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/a392c672d5fa43bc99c0ba468de7441d 2024-11-14T21:33:04,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/534b1b6051684c828297718a1994060e to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/534b1b6051684c828297718a1994060e 2024-11-14T21:33:04,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/2dc60a577e6d490c9fc75ed7a6751e91 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/2dc60a577e6d490c9fc75ed7a6751e91 2024-11-14T21:33:04,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/48ecec4ff74c42b4a075a03c47b5fe31 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/48ecec4ff74c42b4a075a03c47b5fe31 2024-11-14T21:33:04,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/2736e25b5c4d4cdebd367226207c9e44 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/2736e25b5c4d4cdebd367226207c9e44 2024-11-14T21:33:04,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/86b23d4baec24d389b6fa9b7e96640c2 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/86b23d4baec24d389b6fa9b7e96640c2 2024-11-14T21:33:04,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/214ec4a239b745eb96ba46289a0cdf00 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/214ec4a239b745eb96ba46289a0cdf00 2024-11-14T21:33:04,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/1609cb128a3a45849cbd19b0a5c4daaf to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/1609cb128a3a45849cbd19b0a5c4daaf 2024-11-14T21:33:04,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/88185135041d4ceca00a0db9515cd1db, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/5928ea884aba4d1ea1445a10b100acf8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/f7b667a1edb64a6c986552b1a4938154, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/875f29448da3434480797e1db2bb54fa, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/21d65e9621154d74a92ca92015da04c9, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/d48c8792a65546f6902601a3dc256036, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/e8fbddd381a14832b032bf70e576e127, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/9130a4fb1d3646e4ba81bf71fd146dab, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/bf92c06cd4504ef6a392eb62df1c7c08, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/b7cd361413f34fdbbe1722194ed95de6, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/ae20f27606a34cea88e7f20375ee8566, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/9d8e6571a3624ee3aa11980a8808e9c3, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/a91557c939f44e6bbb1958aa8e8e2820, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/aa01ef10bf964156adb19ce62f7e1c1c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/96bf6b9be1d041e3b023327fc91f2a3f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/bd8f17c4b64f4570b4ed1dc0ca4046c0, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/bc097eea7f7e42189d060f36c47863a4, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/a6c872f2a6854f7bb14f92ad3d317646, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/72105dd4cd3e4743a84e55caddb53703] to archive 2024-11-14T21:33:04,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:33:04,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/88185135041d4ceca00a0db9515cd1db to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/88185135041d4ceca00a0db9515cd1db 2024-11-14T21:33:04,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/5928ea884aba4d1ea1445a10b100acf8 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/5928ea884aba4d1ea1445a10b100acf8 2024-11-14T21:33:04,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/f7b667a1edb64a6c986552b1a4938154 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/f7b667a1edb64a6c986552b1a4938154 2024-11-14T21:33:04,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/875f29448da3434480797e1db2bb54fa to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/875f29448da3434480797e1db2bb54fa 2024-11-14T21:33:04,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/21d65e9621154d74a92ca92015da04c9 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/21d65e9621154d74a92ca92015da04c9 2024-11-14T21:33:04,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/d48c8792a65546f6902601a3dc256036 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/d48c8792a65546f6902601a3dc256036 2024-11-14T21:33:04,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/e8fbddd381a14832b032bf70e576e127 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/e8fbddd381a14832b032bf70e576e127 2024-11-14T21:33:04,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/9130a4fb1d3646e4ba81bf71fd146dab to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/9130a4fb1d3646e4ba81bf71fd146dab 2024-11-14T21:33:04,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/bf92c06cd4504ef6a392eb62df1c7c08 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/bf92c06cd4504ef6a392eb62df1c7c08 2024-11-14T21:33:04,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/b7cd361413f34fdbbe1722194ed95de6 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/b7cd361413f34fdbbe1722194ed95de6 2024-11-14T21:33:04,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/ae20f27606a34cea88e7f20375ee8566 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/ae20f27606a34cea88e7f20375ee8566 2024-11-14T21:33:04,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/9d8e6571a3624ee3aa11980a8808e9c3 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/9d8e6571a3624ee3aa11980a8808e9c3 2024-11-14T21:33:04,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/a91557c939f44e6bbb1958aa8e8e2820 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/a91557c939f44e6bbb1958aa8e8e2820 2024-11-14T21:33:04,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/aa01ef10bf964156adb19ce62f7e1c1c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/aa01ef10bf964156adb19ce62f7e1c1c 2024-11-14T21:33:04,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/96bf6b9be1d041e3b023327fc91f2a3f to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/96bf6b9be1d041e3b023327fc91f2a3f 2024-11-14T21:33:04,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/bd8f17c4b64f4570b4ed1dc0ca4046c0 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/bd8f17c4b64f4570b4ed1dc0ca4046c0 2024-11-14T21:33:04,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/bc097eea7f7e42189d060f36c47863a4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/bc097eea7f7e42189d060f36c47863a4 2024-11-14T21:33:04,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/a6c872f2a6854f7bb14f92ad3d317646 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/a6c872f2a6854f7bb14f92ad3d317646 2024-11-14T21:33:04,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/72105dd4cd3e4743a84e55caddb53703 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/72105dd4cd3e4743a84e55caddb53703 2024-11-14T21:33:04,758 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/recovered.edits/306.seqid, newMaxSeqId=306, maxSeqId=4 2024-11-14T21:33:04,759 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. 2024-11-14T21:33:04,759 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1676): Region close journal for 637fa2090a72f9209083e48908b752fe: Waiting for close lock at 1731619982585Running coprocessor pre-close hooks at 1731619982585Disabling compacts and flushes for region at 1731619982585Disabling writes for close at 1731619982885 (+300 ms)Obtaining lock to block concurrent updates at 1731619982885Preparing flush snapshotting stores in 637fa2090a72f9209083e48908b752fe at 1731619982885Finished memstore snapshotting TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731619982886 (+1 ms)Flushing stores of TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe. at 1731619982886Flushing 637fa2090a72f9209083e48908b752fe/A: creating writer at 1731619982886Flushing mob file 637fa2090a72f9209083e48908b752fe/A: appending metadata at 1731619982890 (+4 ms)Flushing mob file 637fa2090a72f9209083e48908b752fe/A: closing flushed file at 1731619982890Flushing 637fa2090a72f9209083e48908b752fe/A: appending metadata at 1731619983300 (+410 ms)Flushing 637fa2090a72f9209083e48908b752fe/A: closing flushed file at 1731619983300Flushing 637fa2090a72f9209083e48908b752fe/B: creating writer at 1731619983710 (+410 ms)Flushing 637fa2090a72f9209083e48908b752fe/B: appending metadata at 1731619983713 (+3 ms)Flushing 637fa2090a72f9209083e48908b752fe/B: closing flushed file at 1731619983713Flushing 637fa2090a72f9209083e48908b752fe/C: creating writer at 1731619984123 (+410 ms)Flushing 637fa2090a72f9209083e48908b752fe/C: appending metadata at 1731619984126 (+3 ms)Flushing 637fa2090a72f9209083e48908b752fe/C: closing flushed file at 1731619984126Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d3d32b: reopening flushed file at 1731619984545 (+419 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a21d4e: reopening flushed file at 1731619984551 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ab9f190: reopening flushed file at 1731619984563 (+12 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 637fa2090a72f9209083e48908b752fe in 1721ms, sequenceid=303, compaction requested=true at 1731619984607 (+44 ms)Writing region close event to WAL at 1731619984753 (+146 ms)Running coprocessor post-close hooks at 1731619984759 (+6 ms)Closed at 1731619984759 2024-11-14T21:33:04,761 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] handler.UnassignRegionHandler(157): Closed 637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:04,761 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=48 updating hbase:meta row=637fa2090a72f9209083e48908b752fe, regionState=CLOSED 2024-11-14T21:33:04,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=48, state=RUNNABLE, hasLock=false; CloseRegionProcedure 637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 because future has completed 2024-11-14T21:33:04,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-14T21:33:04,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; CloseRegionProcedure 637fa2090a72f9209083e48908b752fe, server=ef788a4cade8,37683,1731619918560 in 2.3330 sec 2024-11-14T21:33:04,767 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=47 2024-11-14T21:33:04,767 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=47, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=637fa2090a72f9209083e48908b752fe, UNASSIGN in 2.3380 sec 2024-11-14T21:33:04,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-14T21:33:04,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3430 sec 2024-11-14T21:33:04,771 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731619984771"}]},"ts":"1731619984771"} 2024-11-14T21:33:04,773 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T21:33:04,773 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T21:33:04,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.3570 sec 2024-11-14T21:33:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T21:33:06,580 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:33:06,581 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T21:33:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:33:06,583 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=50, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:33:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T21:33:06,584 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=50, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:33:06,587 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:06,591 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A, FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B, FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C, FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/recovered.edits] 2024-11-14T21:33:06,594 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/21b133e3571f4fc8a99a6c5a6fba3f5c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/21b133e3571f4fc8a99a6c5a6fba3f5c 2024-11-14T21:33:06,596 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/78a4c3ed292541f6abcb76384b0ef835 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/78a4c3ed292541f6abcb76384b0ef835 2024-11-14T21:33:06,597 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/8814e09fe9364057adab19c7c572fe29 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/8814e09fe9364057adab19c7c572fe29 2024-11-14T21:33:06,598 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/a0d1c73988dc4c8baa33e2d282cee1dd to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/A/a0d1c73988dc4c8baa33e2d282cee1dd 2024-11-14T21:33:06,602 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/17c90e081ee6417d9fb1582c9ac04523 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/17c90e081ee6417d9fb1582c9ac04523 2024-11-14T21:33:06,603 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/40bb7cd5d3e74ab48d684c1440bd601e to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/40bb7cd5d3e74ab48d684c1440bd601e 2024-11-14T21:33:06,604 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/eaa947a186bb45aba359ca24dc5f34f2 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/eaa947a186bb45aba359ca24dc5f34f2 2024-11-14T21:33:06,606 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/fd348512253f427f8bee9ba70621c65c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/B/fd348512253f427f8bee9ba70621c65c 2024-11-14T21:33:06,609 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/03c8cb75a4ac4e8288778921b0d0dea5 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/03c8cb75a4ac4e8288778921b0d0dea5 2024-11-14T21:33:06,611 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/ade086ec1a2c4b0bb299007c071ad353 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/ade086ec1a2c4b0bb299007c071ad353 2024-11-14T21:33:06,612 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/d0ed8dd21b3e4154a4e121aac5a83ca8 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/d0ed8dd21b3e4154a4e121aac5a83ca8 2024-11-14T21:33:06,613 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/f001e70616464042a1b75d43a5f74ccb to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/C/f001e70616464042a1b75d43a5f74ccb 2024-11-14T21:33:06,617 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/recovered.edits/306.seqid to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe/recovered.edits/306.seqid 2024-11-14T21:33:06,617 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:06,618 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T21:33:06,618 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T21:33:06,619 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T21:33:06,623 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141f83978d565e4d88b57273008ce6337c_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141f83978d565e4d88b57273008ce6337c_637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:06,625 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143f5de4f770744b279b315844ecbde134_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143f5de4f770744b279b315844ecbde134_637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:06,626 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144c3ebd51337c4485b69e88a10172841a_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144c3ebd51337c4485b69e88a10172841a_637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:06,628 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144e2037c6bd1041ab923dfe17361f3ef7_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144e2037c6bd1041ab923dfe17361f3ef7_637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:06,632 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146206923c6fd34e5a82729bee9d96badf_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146206923c6fd34e5a82729bee9d96badf_637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:06,634 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114744cd127dc9a41aea36b5aef97f4c37d_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114744cd127dc9a41aea36b5aef97f4c37d_637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:06,635 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111477aef402d9a84f169c7225eff8001d1b_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111477aef402d9a84f169c7225eff8001d1b_637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:06,636 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148059238a53fc4b25a02795b3e9e2394e_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148059238a53fc4b25a02795b3e9e2394e_637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:06,638 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148de9105c1991488fbf1ccbd23ed82059_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148de9105c1991488fbf1ccbd23ed82059_637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:06,639 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111494a6489d011c4a7bb48e25acd8f041c7_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111494a6489d011c4a7bb48e25acd8f041c7_637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:06,641 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a9bc76beab6f4fafa3198c1f8d0ab2c9_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a9bc76beab6f4fafa3198c1f8d0ab2c9_637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:06,642 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ad01cc5033754737b0aefdb720f15d94_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ad01cc5033754737b0aefdb720f15d94_637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:06,644 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ae28d81d35b945e585aff8cbc4e21e17_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ae28d81d35b945e585aff8cbc4e21e17_637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:06,645 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114afd14bef5ecd4451bd8e35c5cf815a1a_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114afd14bef5ecd4451bd8e35c5cf815a1a_637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:06,646 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bc27d8aa35234cce8dc4c11403adaed8_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bc27d8aa35234cce8dc4c11403adaed8_637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:06,647 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c3190665f4274d9e8d1253cb5ed642f9_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c3190665f4274d9e8d1253cb5ed642f9_637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:06,649 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f352bbc5bea4426ebd6e54c3f6fe0f5c_637fa2090a72f9209083e48908b752fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f352bbc5bea4426ebd6e54c3f6fe0f5c_637fa2090a72f9209083e48908b752fe 2024-11-14T21:33:06,650 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T21:33:06,653 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=50, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:33:06,656 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T21:33:06,659 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T21:33:06,660 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=50, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:33:06,660 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T21:33:06,660 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731619986660"}]},"ts":"9223372036854775807"} 2024-11-14T21:33:06,663 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T21:33:06,663 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 637fa2090a72f9209083e48908b752fe, NAME => 'TestAcidGuarantees,,1731619956556.637fa2090a72f9209083e48908b752fe.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T21:33:06,663 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T21:33:06,663 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731619986663"}]},"ts":"9223372036854775807"} 2024-11-14T21:33:06,665 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T21:33:06,666 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=50, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:33:06,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 85 msec 2024-11-14T21:33:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T21:33:06,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T21:33:06,694 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:33:06,701 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249 (was 247) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/cluster_a38eb657-3cf0-cdbb-aff3-937f25906809/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_967508010_22 at /127.0.0.1:51204 [Waiting for operation #249] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1436642103_22 at /127.0.0.1:39334 [Waiting for operation #210] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/cluster_a38eb657-3cf0-cdbb-aff3-937f25906809/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=902 (was 809) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=884 (was 1177) 2024-11-14T21:33:06,713 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=902, ProcessCount=11, AvailableMemoryMB=882 2024-11-14T21:33:06,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T21:33:06,715 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T21:33:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T21:33:06,717 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T21:33:06,717 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:06,717 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 51 2024-11-14T21:33:06,718 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T21:33:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-14T21:33:06,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742042_1218 (size=1061) 2024-11-14T21:33:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-14T21:33:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-14T21:33:07,125 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275 2024-11-14T21:33:07,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742043_1219 (size=53) 2024-11-14T21:33:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-14T21:33:07,541 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:33:07,541 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 99a2a424d1830a6f519f1971fbc9bf41, disabling compactions & flushes 2024-11-14T21:33:07,541 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:07,541 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:07,541 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. after waiting 0 ms 2024-11-14T21:33:07,541 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:07,541 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:07,541 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 99a2a424d1830a6f519f1971fbc9bf41: Waiting for close lock at 1731619987541Disabling compacts and flushes for region at 1731619987541Disabling writes for close at 1731619987541Writing region close event to WAL at 1731619987541Closed at 1731619987541 2024-11-14T21:33:07,543 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T21:33:07,543 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731619987543"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731619987543"}]},"ts":"1731619987543"} 2024-11-14T21:33:07,547 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T21:33:07,548 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T21:33:07,548 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731619987548"}]},"ts":"1731619987548"} 2024-11-14T21:33:07,549 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T21:33:07,550 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=99a2a424d1830a6f519f1971fbc9bf41, ASSIGN}] 2024-11-14T21:33:07,551 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=99a2a424d1830a6f519f1971fbc9bf41, ASSIGN 2024-11-14T21:33:07,551 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=99a2a424d1830a6f519f1971fbc9bf41, ASSIGN; state=OFFLINE, location=ef788a4cade8,37683,1731619918560; forceNewPlan=false, retain=false 2024-11-14T21:33:07,702 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=99a2a424d1830a6f519f1971fbc9bf41, regionState=OPENING, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:07,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=99a2a424d1830a6f519f1971fbc9bf41, ASSIGN because future has completed 2024-11-14T21:33:07,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560}] 2024-11-14T21:33:07,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-14T21:33:07,860 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:07,860 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:33:07,860 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:07,860 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:33:07,860 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:07,860 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:07,862 INFO [StoreOpener-99a2a424d1830a6f519f1971fbc9bf41-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:07,863 INFO [StoreOpener-99a2a424d1830a6f519f1971fbc9bf41-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:33:07,863 INFO [StoreOpener-99a2a424d1830a6f519f1971fbc9bf41-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99a2a424d1830a6f519f1971fbc9bf41 columnFamilyName A 2024-11-14T21:33:07,863 DEBUG [StoreOpener-99a2a424d1830a6f519f1971fbc9bf41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:07,863 INFO [StoreOpener-99a2a424d1830a6f519f1971fbc9bf41-1 {}] regionserver.HStore(327): Store=99a2a424d1830a6f519f1971fbc9bf41/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:33:07,863 INFO [StoreOpener-99a2a424d1830a6f519f1971fbc9bf41-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:07,864 INFO [StoreOpener-99a2a424d1830a6f519f1971fbc9bf41-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:33:07,865 INFO [StoreOpener-99a2a424d1830a6f519f1971fbc9bf41-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99a2a424d1830a6f519f1971fbc9bf41 columnFamilyName B 2024-11-14T21:33:07,865 DEBUG [StoreOpener-99a2a424d1830a6f519f1971fbc9bf41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:07,865 INFO [StoreOpener-99a2a424d1830a6f519f1971fbc9bf41-1 {}] regionserver.HStore(327): Store=99a2a424d1830a6f519f1971fbc9bf41/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:33:07,865 INFO [StoreOpener-99a2a424d1830a6f519f1971fbc9bf41-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:07,866 INFO [StoreOpener-99a2a424d1830a6f519f1971fbc9bf41-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:33:07,866 INFO [StoreOpener-99a2a424d1830a6f519f1971fbc9bf41-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99a2a424d1830a6f519f1971fbc9bf41 columnFamilyName C 2024-11-14T21:33:07,866 DEBUG [StoreOpener-99a2a424d1830a6f519f1971fbc9bf41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:07,867 INFO [StoreOpener-99a2a424d1830a6f519f1971fbc9bf41-1 {}] regionserver.HStore(327): Store=99a2a424d1830a6f519f1971fbc9bf41/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:33:07,867 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:07,867 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:07,868 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:07,868 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:07,869 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:07,869 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:07,870 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T21:33:07,871 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:07,873 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T21:33:07,873 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 99a2a424d1830a6f519f1971fbc9bf41; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68855778, jitterRate=0.026031047105789185}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T21:33:07,873 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:07,874 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 99a2a424d1830a6f519f1971fbc9bf41: Running coprocessor pre-open hook at 1731619987860Writing region info on filesystem at 1731619987860Initializing all the Stores at 1731619987861 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731619987861Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731619987861Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731619987861Cleaning up temporary data from old regions at 1731619987869 (+8 ms)Running coprocessor post-open hooks at 1731619987873 (+4 ms)Region opened successfully at 1731619987874 (+1 ms) 2024-11-14T21:33:07,874 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., pid=53, masterSystemTime=1731619987856 2024-11-14T21:33:07,876 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:07,876 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:07,877 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=99a2a424d1830a6f519f1971fbc9bf41, regionState=OPEN, openSeqNum=2, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:07,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 because future has completed 2024-11-14T21:33:07,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-14T21:33:07,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; OpenRegionProcedure 99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 in 175 msec 2024-11-14T21:33:07,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-11-14T21:33:07,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=99a2a424d1830a6f519f1971fbc9bf41, ASSIGN in 330 msec 2024-11-14T21:33:07,883 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T21:33:07,884 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731619987883"}]},"ts":"1731619987883"} 2024-11-14T21:33:07,885 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T21:33:07,886 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T21:33:07,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1710 sec 2024-11-14T21:33:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-14T21:33:08,863 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:33:08,863 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78c64d64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,863 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:33:08,864 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:33:08,865 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:33:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:33:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:33:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39dacfa0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:33:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:33:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:08,867 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:33:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a57c12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,869 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:33:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:33:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:33:08,872 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:33:08,873 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T21:33:08,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is ef788a4cade8,38255,1731619917503 2024-11-14T21:33:08,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@457da5c4 2024-11-14T21:33:08,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T21:33:08,875 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T21:33:08,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69918e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,876 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:33:08,877 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:33:08,877 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:33:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:33:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:33:08,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@657a1c40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:33:08,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:33:08,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:08,879 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:33:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d703dcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,882 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f42fbd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,882 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:33:08,882 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:33:08,883 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:33:08,883 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:33:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:33:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@967eb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:33:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:33:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:08,885 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:33:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@91b74e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,886 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@491abcef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,886 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:33:08,886 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:33:08,887 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:33:08,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:33:08,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:33:08,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ad540b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:33:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:33:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:08,889 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:33:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@661f671b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@740e5304, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,903 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:33:08,904 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:33:08,905 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:33:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:33:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:33:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@373b77d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:33:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:33:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:08,908 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:33:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8624e33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a5bd173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:33:08,910 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:33:08,911 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:33:08,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:33:08,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:33:08,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:08,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55f8f70a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,912 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:33:08,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:33:08,914 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:33:08,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a41641f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27f0faef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,919 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:33:08,920 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:33:08,920 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:33:08,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:33:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:33:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9a8ea6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:33:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:33:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:08,923 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:33:08,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62c9b861, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@424e9fb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,935 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:33:08,935 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:33:08,939 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:33:08,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:33:08,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:33:08,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5721a90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:33:08,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:33:08,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:08,941 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:33:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b497f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,942 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2252b0b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,942 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:33:08,943 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:33:08,944 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:33:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:33:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:33:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3de174d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:33:08,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:33:08,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:08,945 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:33:08,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@544e2e6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14f982f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,947 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:33:08,947 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:33:08,948 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:33:08,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:33:08,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:33:08,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4926bc92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:33:08,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:33:08,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:08,950 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:33:08,950 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b4fb98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16365c37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,951 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:33:08,951 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:33:08,952 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:33:08,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:33:08,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:33:08,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4090beac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:33:08,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:33:08,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:08,953 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:33:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16ad1ccd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:08,954 DEBUG [Thread-1048 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:33:08,956 DEBUG [Thread-1045 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:33:08,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:33:08,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:33:08,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:33:08,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:33:08,995 DEBUG [Thread-1056 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:33:08,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:33:08,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:33:08,995 DEBUG [Thread-1050 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:33:08,995 DEBUG [Thread-1054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:33:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:33:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:33:08,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:33:08,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:33:08,997 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:33:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:33:08,998 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:33:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:33:08,998 DEBUG [Thread-1054 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:33:08,998 DEBUG [Thread-1054 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:33:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:33:09,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:33:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:33:09,001 DEBUG [Thread-1039 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:33:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:33:09,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:33:09,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:33:09,004 DEBUG [Thread-1041 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:33:09,004 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:33:09,005 DEBUG [Thread-1052 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:33:09,005 DEBUG [Thread-1041 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:33:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:33:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:33:09,006 DEBUG [Thread-1041 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:33:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:33:09,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:33:09,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:33:09,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:33:09,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:33:09,014 DEBUG [Thread-1037 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:33:09,015 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:33:09,015 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:33:09,016 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:33:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:33:09,027 DEBUG [Thread-1043 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:33:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:09,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T21:33:09,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:33:09,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:33:09,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:09,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:09,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:09,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:09,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:09,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:09,032 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:33:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-14T21:33:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T21:33:09,034 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:33:09,035 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:33:09,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:33:09,036 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:33:09,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:33:09,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/ef83ecfe35104a9a98f1b9ce1cc76afe is 50, key is test_row_0/A:col10/1731619989027/Put/seqid=0 2024-11-14T21:33:09,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742044_1220 (size=12001) 2024-11-14T21:33:09,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/ef83ecfe35104a9a98f1b9ce1cc76afe 2024-11-14T21:33:09,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731619989044, max timestamp=1731619989138Num uniques 859; 2024-11-14T21:33:09,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731619989044, max timestamp=1731619989138Num uniques 859; 2024-11-14T21:33:09,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:33:09,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:33:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T21:33:09,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731619989130, max timestamp=1731619989138Num uniques 859; 2024-11-14T21:33:09,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:33:09,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/a9a04497da4842c8a3505e80fc4d68b7 is 50, key is test_row_0/B:col10/1731619989027/Put/seqid=0 2024-11-14T21:33:09,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T21:33:09,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:09,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:09,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:09,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:09,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:09,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:09,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620049184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:09,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:09,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620049195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:09,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:09,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:09,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620049195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:09,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:09,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1731620049196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:09,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742045_1221 (size=12001) 2024-11-14T21:33:09,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46318 deadline: 1731620049198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:09,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/a9a04497da4842c8a3505e80fc4d68b7 2024-11-14T21:33:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:09,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/8191f793f2184ffda47c92bdecc174f5 is 50, key is test_row_0/C:col10/1731619989027/Put/seqid=0 2024-11-14T21:33:09,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742046_1222 (size=12001) 2024-11-14T21:33:09,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620049315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:09,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:09,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1731620049315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:09,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620049317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:09,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:09,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46318 deadline: 1731620049317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:09,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:09,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620049315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:09,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:09,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T21:33:09,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:09,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:09,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:09,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:09,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T21:33:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:09,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T21:33:09,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:09,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:09,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:09,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:09,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:09,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620049531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:09,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46318 deadline: 1731620049531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:09,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:09,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:09,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620049531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:09,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:09,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:09,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1731620049533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:09,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:09,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:09,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620049540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:09,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T21:33:09,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T21:33:09,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:09,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:09,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:09,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:09,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:09,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/8191f793f2184ffda47c92bdecc174f5 2024-11-14T21:33:09,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/ef83ecfe35104a9a98f1b9ce1cc76afe as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/ef83ecfe35104a9a98f1b9ce1cc76afe 2024-11-14T21:33:09,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/ef83ecfe35104a9a98f1b9ce1cc76afe, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T21:33:09,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/a9a04497da4842c8a3505e80fc4d68b7 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a9a04497da4842c8a3505e80fc4d68b7 2024-11-14T21:33:09,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a9a04497da4842c8a3505e80fc4d68b7, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T21:33:09,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/8191f793f2184ffda47c92bdecc174f5 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/8191f793f2184ffda47c92bdecc174f5 2024-11-14T21:33:09,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/8191f793f2184ffda47c92bdecc174f5, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T21:33:09,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for 99a2a424d1830a6f519f1971fbc9bf41 in 735ms, sequenceid=12, compaction requested=false 2024-11-14T21:33:09,762 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T21:33:09,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:09,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T21:33:09,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:09,847 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=129.71 KB heapSize=365.45 KB 2024-11-14T21:33:09,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:09,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:09,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:09,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:09,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:09,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:09,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:09,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:09,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620049861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:09,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:09,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46318 deadline: 1731620049869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:09,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/4f33fa600c01405aa40dc89ac6b1bf93 is 50, key is test_row_0/A:col10/1731619989163/Put/seqid=0 2024-11-14T21:33:09,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:09,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620049869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:09,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:09,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:09,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1731620049885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:09,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:09,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:09,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620049894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:09,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:09,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742047_1223 (size=14341) 2024-11-14T21:33:09,962 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/4f33fa600c01405aa40dc89ac6b1bf93 2024-11-14T21:33:09,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:09,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620049980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:09,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:09,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620050001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1731620050001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46318 deadline: 1731620050002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/c76c463bdcf2437f94dae27af3fb0414 is 50, key is test_row_0/B:col10/1731619989163/Put/seqid=0 2024-11-14T21:33:10,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620050023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742048_1224 (size=12001) 2024-11-14T21:33:10,069 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/c76c463bdcf2437f94dae27af3fb0414 2024-11-14T21:33:10,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/8fca43e987984610a384b3820b474e31 is 50, key is test_row_0/C:col10/1731619989163/Put/seqid=0 2024-11-14T21:33:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T21:33:10,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742049_1225 (size=12001) 2024-11-14T21:33:10,195 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/8fca43e987984610a384b3820b474e31 2024-11-14T21:33:10,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620050201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/4f33fa600c01405aa40dc89ac6b1bf93 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/4f33fa600c01405aa40dc89ac6b1bf93 2024-11-14T21:33:10,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1731620050223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620050223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46318 deadline: 1731620050223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,234 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/4f33fa600c01405aa40dc89ac6b1bf93, entries=200, sequenceid=48, filesize=14.0 K 2024-11-14T21:33:10,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/c76c463bdcf2437f94dae27af3fb0414 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/c76c463bdcf2437f94dae27af3fb0414 2024-11-14T21:33:10,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620050242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,285 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/c76c463bdcf2437f94dae27af3fb0414, entries=150, sequenceid=48, filesize=11.7 K 2024-11-14T21:33:10,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/8fca43e987984610a384b3820b474e31 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/8fca43e987984610a384b3820b474e31 2024-11-14T21:33:10,311 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T21:33:10,313 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/8fca43e987984610a384b3820b474e31, entries=150, sequenceid=48, filesize=11.7 K 2024-11-14T21:33:10,314 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=53.67 KB/54960 for 99a2a424d1830a6f519f1971fbc9bf41 in 467ms, sequenceid=48, compaction requested=false 2024-11-14T21:33:10,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:10,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:10,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-14T21:33:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-14T21:33:10,334 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-14T21:33:10,334 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2920 sec 2024-11-14T21:33:10,344 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.3080 sec 2024-11-14T21:33:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:10,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T21:33:10,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:10,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:10,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:10,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:10,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:10,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:10,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/52ba67a38c9549a4838d78aad00080f3 is 50, key is test_row_0/A:col10/1731619989873/Put/seqid=0 2024-11-14T21:33:10,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742050_1226 (size=12001) 2024-11-14T21:33:10,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731619990580, max timestamp=1731619990588Num uniques 859; 2024-11-14T21:33:10,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:33:10,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731619990580, max timestamp=1731619990588Num uniques 859; 2024-11-14T21:33:10,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:33:10,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731619990580, max timestamp=1731619990588Num uniques 859; 2024-11-14T21:33:10,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:33:10,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1731620050614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620050617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620050619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46318 deadline: 1731620050630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620050639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1731620050730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620050743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620050743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46318 deadline: 1731620050750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620050760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1731620050944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620050950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620050966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46318 deadline: 1731620050970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:10,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620050972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:10,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:10,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:10,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/52ba67a38c9549a4838d78aad00080f3 2024-11-14T21:33:11,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/8842380cce6b4e33b19e0cda90a9c9a3 is 50, key is test_row_0/B:col10/1731619989873/Put/seqid=0 2024-11-14T21:33:11,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742051_1227 (size=12001) 2024-11-14T21:33:11,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/8842380cce6b4e33b19e0cda90a9c9a3 2024-11-14T21:33:11,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/3feb8c2082374469973f7c86e0ac2c6e is 50, key is test_row_0/C:col10/1731619989873/Put/seqid=0 2024-11-14T21:33:11,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742052_1228 (size=12001) 2024-11-14T21:33:11,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/3feb8c2082374469973f7c86e0ac2c6e 2024-11-14T21:33:11,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/52ba67a38c9549a4838d78aad00080f3 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/52ba67a38c9549a4838d78aad00080f3 2024-11-14T21:33:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T21:33:11,191 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:33:11,199 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:33:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-14T21:33:11,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/52ba67a38c9549a4838d78aad00080f3, entries=150, sequenceid=60, filesize=11.7 K 2024-11-14T21:33:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T21:33:11,201 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:33:11,202 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:33:11,202 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:33:11,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/8842380cce6b4e33b19e0cda90a9c9a3 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/8842380cce6b4e33b19e0cda90a9c9a3 2024-11-14T21:33:11,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/8842380cce6b4e33b19e0cda90a9c9a3, entries=150, sequenceid=60, filesize=11.7 K 2024-11-14T21:33:11,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/3feb8c2082374469973f7c86e0ac2c6e as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/3feb8c2082374469973f7c86e0ac2c6e 2024-11-14T21:33:11,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/3feb8c2082374469973f7c86e0ac2c6e, entries=150, sequenceid=60, filesize=11.7 K 2024-11-14T21:33:11,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 99a2a424d1830a6f519f1971fbc9bf41 in 698ms, sequenceid=60, compaction requested=true 2024-11-14T21:33:11,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:11,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:33:11,238 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:11,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:11,238 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:11,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:33:11,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:11,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:33:11,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:11,240 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:11,240 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:11,240 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/A is initiating minor compaction (all files) 2024-11-14T21:33:11,240 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/B is initiating minor compaction (all files) 2024-11-14T21:33:11,241 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/A in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:11,241 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/ef83ecfe35104a9a98f1b9ce1cc76afe, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/4f33fa600c01405aa40dc89ac6b1bf93, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/52ba67a38c9549a4838d78aad00080f3] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=37.4 K 2024-11-14T21:33:11,241 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/B in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:11,241 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a9a04497da4842c8a3505e80fc4d68b7, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/c76c463bdcf2437f94dae27af3fb0414, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/8842380cce6b4e33b19e0cda90a9c9a3] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=35.2 K 2024-11-14T21:33:11,241 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef83ecfe35104a9a98f1b9ce1cc76afe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731619989010 2024-11-14T21:33:11,242 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting a9a04497da4842c8a3505e80fc4d68b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731619989010 2024-11-14T21:33:11,242 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f33fa600c01405aa40dc89ac6b1bf93, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731619989163 2024-11-14T21:33:11,242 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52ba67a38c9549a4838d78aad00080f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731619989863 2024-11-14T21:33:11,242 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting c76c463bdcf2437f94dae27af3fb0414, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731619989163 2024-11-14T21:33:11,243 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 8842380cce6b4e33b19e0cda90a9c9a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731619989863 2024-11-14T21:33:11,259 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#A#compaction#188 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:11,260 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/7fe67b9e8fb14da2b9175c7d851397ff is 50, key is test_row_0/A:col10/1731619989873/Put/seqid=0 2024-11-14T21:33:11,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-14T21:33:11,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:11,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:11,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:11,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:11,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:11,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:11,269 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#B#compaction#189 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:11,269 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/a94e00f7f574456a8734465a265692fe is 50, key is test_row_0/B:col10/1731619989873/Put/seqid=0 2024-11-14T21:33:11,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/617291120c74496da68943fdce46ddfe is 50, key is test_row_0/A:col10/1731619990625/Put/seqid=0 2024-11-14T21:33:11,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620051286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:11,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:11,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1731620051287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:11,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:11,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620051291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:11,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:11,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46318 deadline: 1731620051293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:11,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:11,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620051294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:11,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T21:33:11,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742053_1229 (size=12104) 2024-11-14T21:33:11,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742054_1230 (size=12104) 2024-11-14T21:33:11,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T21:33:11,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:11,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:11,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:11,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:11,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:11,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742055_1231 (size=14341) 2024-11-14T21:33:11,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/617291120c74496da68943fdce46ddfe 2024-11-14T21:33:11,363 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/a94e00f7f574456a8734465a265692fe as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a94e00f7f574456a8734465a265692fe 2024-11-14T21:33:11,376 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/B of 99a2a424d1830a6f519f1971fbc9bf41 into a94e00f7f574456a8734465a265692fe(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:11,376 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:11,376 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/B, priority=13, startTime=1731619991238; duration=0sec 2024-11-14T21:33:11,376 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:11,376 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:B 2024-11-14T21:33:11,376 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:11,377 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:11,377 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/C is initiating minor compaction (all files) 2024-11-14T21:33:11,377 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/C in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:11,378 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/8191f793f2184ffda47c92bdecc174f5, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/8fca43e987984610a384b3820b474e31, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/3feb8c2082374469973f7c86e0ac2c6e] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=35.2 K 2024-11-14T21:33:11,380 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 8191f793f2184ffda47c92bdecc174f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731619989010 2024-11-14T21:33:11,381 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fca43e987984610a384b3820b474e31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731619989163 2024-11-14T21:33:11,381 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 3feb8c2082374469973f7c86e0ac2c6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731619989863 2024-11-14T21:33:11,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/a446d83fdced4ceca7852886e5ab9ab5 is 50, key is test_row_0/B:col10/1731619990625/Put/seqid=0 2024-11-14T21:33:11,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620051401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:11,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:11,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:11,416 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#C#compaction#192 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46318 deadline: 1731620051410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:11,417 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/bfef5965d5524572885c92e7097e2257 is 50, key is test_row_0/C:col10/1731619989873/Put/seqid=0 2024-11-14T21:33:11,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:11,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1731620051410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:11,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:11,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:11,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620051411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620051411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:11,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:11,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:11,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742056_1232 (size=12001) 2024-11-14T21:33:11,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/a446d83fdced4ceca7852886e5ab9ab5 2024-11-14T21:33:11,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742057_1233 (size=12104) 2024-11-14T21:33:11,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/94bc0023ec2c4858a7cf77b1cb00f6d4 is 50, key is test_row_0/C:col10/1731619990625/Put/seqid=0 2024-11-14T21:33:11,483 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/bfef5965d5524572885c92e7097e2257 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/bfef5965d5524572885c92e7097e2257 2024-11-14T21:33:11,498 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/C of 99a2a424d1830a6f519f1971fbc9bf41 into bfef5965d5524572885c92e7097e2257(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:11,498 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:11,498 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/C, priority=13, startTime=1731619991238; duration=0sec 2024-11-14T21:33:11,498 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:11,498 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:C 2024-11-14T21:33:11,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T21:33:11,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:11,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:11,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:11,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:11,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:11,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742058_1234 (size=12001) 2024-11-14T21:33:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T21:33:11,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/94bc0023ec2c4858a7cf77b1cb00f6d4 2024-11-14T21:33:11,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/617291120c74496da68943fdce46ddfe as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/617291120c74496da68943fdce46ddfe 2024-11-14T21:33:11,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/617291120c74496da68943fdce46ddfe, entries=200, sequenceid=97, filesize=14.0 K 2024-11-14T21:33:11,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/a446d83fdced4ceca7852886e5ab9ab5 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a446d83fdced4ceca7852886e5ab9ab5 2024-11-14T21:33:11,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a446d83fdced4ceca7852886e5ab9ab5, entries=150, sequenceid=97, filesize=11.7 K 2024-11-14T21:33:11,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/94bc0023ec2c4858a7cf77b1cb00f6d4 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/94bc0023ec2c4858a7cf77b1cb00f6d4 2024-11-14T21:33:11,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/94bc0023ec2c4858a7cf77b1cb00f6d4, entries=150, sequenceid=97, filesize=11.7 K 2024-11-14T21:33:11,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 99a2a424d1830a6f519f1971fbc9bf41 in 285ms, sequenceid=97, compaction requested=false 2024-11-14T21:33:11,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:11,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:11,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T21:33:11,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:11,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:11,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:11,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:11,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:11,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:11,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T21:33:11,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:11,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:11,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:11,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:11,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:11,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:11,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/aff7771bdf714466b18483c08dee4bce is 50, key is test_row_0/A:col10/1731619991639/Put/seqid=0 2024-11-14T21:33:11,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742059_1235 (size=12001) 2024-11-14T21:33:11,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:11,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620051696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:11,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731619991666, max timestamp=1731619991705Num uniques 859; 2024-11-14T21:33:11,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T21:33:11,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731619991666, max timestamp=1731619991705Num uniques 859; 2024-11-14T21:33:11,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T21:33:11,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:11,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731619991666, max timestamp=1731619991705Num uniques 859; 2024-11-14T21:33:11,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T21:33:11,742 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/7fe67b9e8fb14da2b9175c7d851397ff as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/7fe67b9e8fb14da2b9175c7d851397ff 2024-11-14T21:33:11,749 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/A of 99a2a424d1830a6f519f1971fbc9bf41 into 7fe67b9e8fb14da2b9175c7d851397ff(size=11.8 K), total size for store is 25.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:11,750 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:11,750 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/A, priority=13, startTime=1731619991238; duration=0sec 2024-11-14T21:33:11,750 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:11,750 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:A 2024-11-14T21:33:11,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620051754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:11,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620051757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:11,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1731620051775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:11,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:11,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46318 deadline: 1731620051776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:11,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T21:33:11,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:11,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:11,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:11,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:11,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:11,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620051823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T21:33:11,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620051882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:11,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:11,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620051891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:11,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:11,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1731620051900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:11,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:11,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46318 deadline: 1731620051902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:11,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T21:33:11,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:11,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:11,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:11,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:11,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:12,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620052034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:12,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620052090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:12,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:12,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/aff7771bdf714466b18483c08dee4bce 2024-11-14T21:33:12,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620052103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:12,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:12,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1731620052113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:12,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46318 deadline: 1731620052113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:12,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:12,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/714b8b9639574954be8fe91a48a1d248 is 50, key is test_row_0/B:col10/1731619991639/Put/seqid=0 2024-11-14T21:33:12,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T21:33:12,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:12,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:12,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:12,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:12,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:12,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742060_1236 (size=12001) 2024-11-14T21:33:12,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T21:33:12,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:12,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:12,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:12,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:12,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:12,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T21:33:12,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620052351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:12,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620052410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:12,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620052421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:12,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1731620052434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:12,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:12,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46318 deadline: 1731620052434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:12,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:12,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T21:33:12,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:12,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:12,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:12,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:12,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:12,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/714b8b9639574954be8fe91a48a1d248 2024-11-14T21:33:12,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T21:33:12,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:12,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:12,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:12,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:12,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:12,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/e1744d2d55e74600a0846d818f31bb9f is 50, key is test_row_0/C:col10/1731619991639/Put/seqid=0 2024-11-14T21:33:12,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742061_1237 (size=12001) 2024-11-14T21:33:12,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/e1744d2d55e74600a0846d818f31bb9f 2024-11-14T21:33:12,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/aff7771bdf714466b18483c08dee4bce as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/aff7771bdf714466b18483c08dee4bce 2024-11-14T21:33:12,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T21:33:12,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:12,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:12,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:12,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:12,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:12,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/aff7771bdf714466b18483c08dee4bce, entries=150, sequenceid=111, filesize=11.7 K 2024-11-14T21:33:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:12,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/714b8b9639574954be8fe91a48a1d248 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/714b8b9639574954be8fe91a48a1d248 2024-11-14T21:33:12,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/714b8b9639574954be8fe91a48a1d248, entries=150, sequenceid=111, filesize=11.7 K 2024-11-14T21:33:12,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/e1744d2d55e74600a0846d818f31bb9f as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/e1744d2d55e74600a0846d818f31bb9f 2024-11-14T21:33:12,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/e1744d2d55e74600a0846d818f31bb9f, entries=150, sequenceid=111, filesize=11.7 K 2024-11-14T21:33:12,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 99a2a424d1830a6f519f1971fbc9bf41 in 1163ms, sequenceid=111, compaction requested=true 2024-11-14T21:33:12,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:12,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:33:12,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:12,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:33:12,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:33:12,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:33:12,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T21:33:12,805 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:12,806 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:12,806 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/A is initiating minor compaction (all files) 2024-11-14T21:33:12,806 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/A in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:12,807 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/7fe67b9e8fb14da2b9175c7d851397ff, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/617291120c74496da68943fdce46ddfe, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/aff7771bdf714466b18483c08dee4bce] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=37.5 K 2024-11-14T21:33:12,808 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fe67b9e8fb14da2b9175c7d851397ff, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731619989863 2024-11-14T21:33:12,808 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:12,809 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 617291120c74496da68943fdce46ddfe, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731619990614 2024-11-14T21:33:12,809 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting aff7771bdf714466b18483c08dee4bce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731619991291 2024-11-14T21:33:12,810 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:12,810 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/C is initiating minor compaction (all files) 2024-11-14T21:33:12,810 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/C in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:12,810 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/bfef5965d5524572885c92e7097e2257, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/94bc0023ec2c4858a7cf77b1cb00f6d4, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/e1744d2d55e74600a0846d818f31bb9f] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=35.3 K 2024-11-14T21:33:12,811 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfef5965d5524572885c92e7097e2257, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731619989863 2024-11-14T21:33:12,812 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94bc0023ec2c4858a7cf77b1cb00f6d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731619990614 2024-11-14T21:33:12,813 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1744d2d55e74600a0846d818f31bb9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731619991291 2024-11-14T21:33:12,842 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#A#compaction#197 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:12,843 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/42cf552ebab946928b08a9590d50a6c6 is 50, key is test_row_0/A:col10/1731619991639/Put/seqid=0 2024-11-14T21:33:12,857 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#C#compaction#198 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:12,857 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/4f52ab1047084866bdec9c9e73660826 is 50, key is test_row_0/C:col10/1731619991639/Put/seqid=0 2024-11-14T21:33:12,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:12,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-14T21:33:12,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:12,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:12,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:12,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:12,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:12,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:12,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T21:33:12,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:12,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742063_1239 (size=12207) 2024-11-14T21:33:12,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:12,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:12,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:12,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:12,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:12,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742062_1238 (size=12207) 2024-11-14T21:33:12,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/2d29c1da398e4e1d803c85a736e4c34e is 50, key is test_row_0/A:col10/1731619991752/Put/seqid=0 2024-11-14T21:33:12,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:12,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1731620052955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:12,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:12,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46318 deadline: 1731620052955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:12,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742064_1240 (size=14541) 2024-11-14T21:33:12,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/2d29c1da398e4e1d803c85a736e4c34e 2024-11-14T21:33:12,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:12,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620052960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:12,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:12,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:12,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620052966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:12,975 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/42cf552ebab946928b08a9590d50a6c6 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/42cf552ebab946928b08a9590d50a6c6 2024-11-14T21:33:12,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620052966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:12,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:12,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:12,983 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/A of 99a2a424d1830a6f519f1971fbc9bf41 into 42cf552ebab946928b08a9590d50a6c6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:12,983 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:12,983 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/A, priority=13, startTime=1731619992805; duration=0sec 2024-11-14T21:33:12,983 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:12,983 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:A 2024-11-14T21:33:12,983 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:12,986 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:12,986 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/B is initiating minor compaction (all files) 2024-11-14T21:33:12,986 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/B in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:12,987 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a94e00f7f574456a8734465a265692fe, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a446d83fdced4ceca7852886e5ab9ab5, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/714b8b9639574954be8fe91a48a1d248] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=35.3 K 2024-11-14T21:33:12,987 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting a94e00f7f574456a8734465a265692fe, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731619989863 2024-11-14T21:33:12,987 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting a446d83fdced4ceca7852886e5ab9ab5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731619990614 2024-11-14T21:33:12,988 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 714b8b9639574954be8fe91a48a1d248, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731619991291 2024-11-14T21:33:12,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/f5f0fde12be14dbba0b0ecff515fbcd8 is 50, key is test_row_0/B:col10/1731619991752/Put/seqid=0 2024-11-14T21:33:13,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742065_1241 (size=12151) 2024-11-14T21:33:13,055 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#B#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:13,057 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/144af65ca3d14775b9889353bc47cb10 is 50, key is test_row_0/B:col10/1731619991639/Put/seqid=0 2024-11-14T21:33:13,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T21:33:13,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:13,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:13,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:13,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:13,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:13,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:13,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:13,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620053083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:13,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:13,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620053083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:13,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:13,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:13,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:13,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:13,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620053083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:13,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742066_1242 (size=12207) 2024-11-14T21:33:13,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T21:33:13,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:13,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:13,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:13,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:13,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:13,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:13,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:13,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620053300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:13,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:13,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:13,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:13,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620053300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:13,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:13,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:13,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620053311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:13,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:13,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:13,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:13,340 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/4f52ab1047084866bdec9c9e73660826 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/4f52ab1047084866bdec9c9e73660826 2024-11-14T21:33:13,347 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/C of 99a2a424d1830a6f519f1971fbc9bf41 into 4f52ab1047084866bdec9c9e73660826(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:13,347 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:13,347 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/C, priority=13, startTime=1731619992805; duration=0sec 2024-11-14T21:33:13,347 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:13,347 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:C 2024-11-14T21:33:13,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T21:33:13,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T21:33:13,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:13,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:13,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:13,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:13,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:13,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:13,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/f5f0fde12be14dbba0b0ecff515fbcd8 2024-11-14T21:33:13,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/940f1587ea2c45baa6e87eae859bc80c is 50, key is test_row_0/C:col10/1731619991752/Put/seqid=0 2024-11-14T21:33:13,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742067_1243 (size=12151) 2024-11-14T21:33:13,535 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/144af65ca3d14775b9889353bc47cb10 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/144af65ca3d14775b9889353bc47cb10 2024-11-14T21:33:13,545 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/B of 99a2a424d1830a6f519f1971fbc9bf41 into 144af65ca3d14775b9889353bc47cb10(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:13,546 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:13,546 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/B, priority=13, startTime=1731619992805; duration=0sec 2024-11-14T21:33:13,546 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:13,546 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:B 2024-11-14T21:33:13,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T21:33:13,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:13,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:13,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:13,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:13,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:13,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:13,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620053614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:13,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:13,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620053620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:13,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620053620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:13,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:13,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:13,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:13,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T21:33:13,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:13,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:13,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:13,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:13,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:13,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:13,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T21:33:13,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:13,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:13,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:13,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:13,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:13,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:13,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/940f1587ea2c45baa6e87eae859bc80c 2024-11-14T21:33:13,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/2d29c1da398e4e1d803c85a736e4c34e as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/2d29c1da398e4e1d803c85a736e4c34e 2024-11-14T21:33:13,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/2d29c1da398e4e1d803c85a736e4c34e, entries=200, sequenceid=147, filesize=14.2 K 2024-11-14T21:33:13,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/f5f0fde12be14dbba0b0ecff515fbcd8 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/f5f0fde12be14dbba0b0ecff515fbcd8 2024-11-14T21:33:13,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:13,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46318 deadline: 1731620053981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:13,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:13,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1731620053982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:13,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/f5f0fde12be14dbba0b0ecff515fbcd8, entries=150, sequenceid=147, filesize=11.9 K 2024-11-14T21:33:13,990 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_0' in 99a2a424d1830a6f519f1971fbc9bf41 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:33:13,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:13,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:13,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:13,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/940f1587ea2c45baa6e87eae859bc80c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/940f1587ea2c45baa6e87eae859bc80c 2024-11-14T21:33:13,994 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_1' in 99a2a424d1830a6f519f1971fbc9bf41 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:33:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:14,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/940f1587ea2c45baa6e87eae859bc80c, entries=150, sequenceid=147, filesize=11.9 K 2024-11-14T21:33:14,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for 99a2a424d1830a6f519f1971fbc9bf41 in 1133ms, sequenceid=147, compaction requested=false 2024-11-14T21:33:14,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:14,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T21:33:14,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:14,029 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T21:33:14,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:14,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:14,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:14,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:14,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:14,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:14,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/03c3453974654ea2b7041ad775be87ce is 50, key is test_row_0/A:col10/1731619992954/Put/seqid=0 2024-11-14T21:33:14,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742068_1244 (size=12151) 2024-11-14T21:33:14,067 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/03c3453974654ea2b7041ad775be87ce 2024-11-14T21:33:14,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/be16da7816b24591a73a1c339ba8f0c9 is 50, key is test_row_0/B:col10/1731619992954/Put/seqid=0 2024-11-14T21:33:14,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:14,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:14,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742069_1245 (size=12151) 2024-11-14T21:33:14,138 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/be16da7816b24591a73a1c339ba8f0c9 2024-11-14T21:33:14,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/bf9824eaf74c4a37b336ab01fb73c67a is 50, key is test_row_0/C:col10/1731619992954/Put/seqid=0 2024-11-14T21:33:14,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620054189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:14,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:14,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742070_1246 (size=12151) 2024-11-14T21:33:14,201 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/bf9824eaf74c4a37b336ab01fb73c67a 2024-11-14T21:33:14,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620054193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:14,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:14,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:14,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620054196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:14,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/03c3453974654ea2b7041ad775be87ce as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/03c3453974654ea2b7041ad775be87ce 2024-11-14T21:33:14,213 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/03c3453974654ea2b7041ad775be87ce, entries=150, sequenceid=162, filesize=11.9 K 2024-11-14T21:33:14,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/be16da7816b24591a73a1c339ba8f0c9 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/be16da7816b24591a73a1c339ba8f0c9 2024-11-14T21:33:14,222 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/be16da7816b24591a73a1c339ba8f0c9, entries=150, sequenceid=162, filesize=11.9 K 2024-11-14T21:33:14,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/bf9824eaf74c4a37b336ab01fb73c67a as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/bf9824eaf74c4a37b336ab01fb73c67a 2024-11-14T21:33:14,229 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/bf9824eaf74c4a37b336ab01fb73c67a, entries=150, sequenceid=162, filesize=11.9 K 2024-11-14T21:33:14,230 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 99a2a424d1830a6f519f1971fbc9bf41 in 201ms, sequenceid=162, compaction requested=true 2024-11-14T21:33:14,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:14,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:14,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-14T21:33:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-14T21:33:14,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-14T21:33:14,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0350 sec 2024-11-14T21:33:14,247 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 3.0450 sec 2024-11-14T21:33:14,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-14T21:33:14,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:14,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:14,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:14,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:14,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:14,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:14,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/ee972a4c9d9145ffa14e68156524646d is 50, key is test_row_0/A:col10/1731619994193/Put/seqid=0 2024-11-14T21:33:14,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742071_1247 (size=12151) 2024-11-14T21:33:14,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620054355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:14,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:14,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620054360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:14,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620054361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:14,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:14,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:14,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:14,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620054473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:14,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:14,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620054473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:14,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:14,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:14,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620054473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:14,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:14,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620054691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:14,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:14,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620054691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:14,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620054691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:14,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:14,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:14,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/ee972a4c9d9145ffa14e68156524646d 2024-11-14T21:33:14,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/77343de8969a4e4c968c155ea1db1a17 is 50, key is test_row_0/B:col10/1731619994193/Put/seqid=0 2024-11-14T21:33:14,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742072_1248 (size=12151) 2024-11-14T21:33:15,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620055000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:15,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620055000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:15,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:15,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620055010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:15,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:15,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:15,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:15,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/77343de8969a4e4c968c155ea1db1a17 2024-11-14T21:33:15,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/a0cc0a64e5ad46fc87a9917cd760d0db is 50, key is test_row_0/C:col10/1731619994193/Put/seqid=0 2024-11-14T21:33:15,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742073_1249 (size=12151) 2024-11-14T21:33:15,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/a0cc0a64e5ad46fc87a9917cd760d0db 2024-11-14T21:33:15,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T21:33:15,382 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:33:15,383 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:33:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-14T21:33:15,388 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:33:15,388 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:33:15,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:33:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T21:33:15,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/ee972a4c9d9145ffa14e68156524646d as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/ee972a4c9d9145ffa14e68156524646d 2024-11-14T21:33:15,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/ee972a4c9d9145ffa14e68156524646d, entries=150, sequenceid=183, filesize=11.9 K 2024-11-14T21:33:15,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/77343de8969a4e4c968c155ea1db1a17 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/77343de8969a4e4c968c155ea1db1a17 2024-11-14T21:33:15,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/77343de8969a4e4c968c155ea1db1a17, entries=150, sequenceid=183, filesize=11.9 K 2024-11-14T21:33:15,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/a0cc0a64e5ad46fc87a9917cd760d0db as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/a0cc0a64e5ad46fc87a9917cd760d0db 2024-11-14T21:33:15,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/a0cc0a64e5ad46fc87a9917cd760d0db, entries=150, sequenceid=183, filesize=11.9 K 2024-11-14T21:33:15,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 99a2a424d1830a6f519f1971fbc9bf41 in 1127ms, sequenceid=183, compaction requested=true 2024-11-14T21:33:15,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:15,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:33:15,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:15,435 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:33:15,435 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:33:15,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:33:15,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:15,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:33:15,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:15,440 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51050 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:33:15,440 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:33:15,440 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/A is initiating minor compaction (all files) 2024-11-14T21:33:15,440 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/B is initiating minor compaction (all files) 2024-11-14T21:33:15,440 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/B in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:15,440 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/A in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:15,440 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/144af65ca3d14775b9889353bc47cb10, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/f5f0fde12be14dbba0b0ecff515fbcd8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/be16da7816b24591a73a1c339ba8f0c9, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/77343de8969a4e4c968c155ea1db1a17] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=47.5 K 2024-11-14T21:33:15,440 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/42cf552ebab946928b08a9590d50a6c6, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/2d29c1da398e4e1d803c85a736e4c34e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/03c3453974654ea2b7041ad775be87ce, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/ee972a4c9d9145ffa14e68156524646d] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=49.9 K 2024-11-14T21:33:15,441 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 144af65ca3d14775b9889353bc47cb10, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731619991291 2024-11-14T21:33:15,441 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42cf552ebab946928b08a9590d50a6c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731619991291 2024-11-14T21:33:15,442 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting f5f0fde12be14dbba0b0ecff515fbcd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731619991752 2024-11-14T21:33:15,442 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d29c1da398e4e1d803c85a736e4c34e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731619991752 2024-11-14T21:33:15,442 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting be16da7816b24591a73a1c339ba8f0c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731619992942 2024-11-14T21:33:15,444 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03c3453974654ea2b7041ad775be87ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731619992942 2024-11-14T21:33:15,444 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 77343de8969a4e4c968c155ea1db1a17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731619994189 2024-11-14T21:33:15,445 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee972a4c9d9145ffa14e68156524646d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731619994189 2024-11-14T21:33:15,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T21:33:15,496 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#B#compaction#209 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:15,497 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/b0525bc3c0a04c9dbbf3f97ed624df46 is 50, key is test_row_0/B:col10/1731619994193/Put/seqid=0 2024-11-14T21:33:15,500 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#A#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:15,501 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/b1a99692dd404dee8fcc23d49696e18c is 50, key is test_row_0/A:col10/1731619994193/Put/seqid=0 2024-11-14T21:33:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:15,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T21:33:15,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:15,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:15,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:15,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:15,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:15,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:15,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742075_1251 (size=12493) 2024-11-14T21:33:15,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742074_1250 (size=12493) 2024-11-14T21:33:15,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/4189428e0b8640bd811c9400e1d45ecc is 50, key is test_row_0/A:col10/1731619994357/Put/seqid=0 2024-11-14T21:33:15,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-14T21:33:15,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:15,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:15,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:15,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:15,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:15,549 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/b1a99692dd404dee8fcc23d49696e18c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/b1a99692dd404dee8fcc23d49696e18c 2024-11-14T21:33:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:15,551 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/b0525bc3c0a04c9dbbf3f97ed624df46 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b0525bc3c0a04c9dbbf3f97ed624df46 2024-11-14T21:33:15,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742076_1252 (size=14541) 2024-11-14T21:33:15,558 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/A of 99a2a424d1830a6f519f1971fbc9bf41 into b1a99692dd404dee8fcc23d49696e18c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:15,558 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:15,558 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/A, priority=12, startTime=1731619995435; duration=0sec 2024-11-14T21:33:15,558 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:15,558 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:A 2024-11-14T21:33:15,558 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:33:15,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/4189428e0b8640bd811c9400e1d45ecc 2024-11-14T21:33:15,578 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/B of 99a2a424d1830a6f519f1971fbc9bf41 into b0525bc3c0a04c9dbbf3f97ed624df46(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:15,578 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:15,578 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/B, priority=12, startTime=1731619995435; duration=0sec 2024-11-14T21:33:15,578 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:15,578 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:B 2024-11-14T21:33:15,579 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:33:15,579 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/C is initiating minor compaction (all files) 2024-11-14T21:33:15,579 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/C in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:15,579 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/4f52ab1047084866bdec9c9e73660826, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/940f1587ea2c45baa6e87eae859bc80c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/bf9824eaf74c4a37b336ab01fb73c67a, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/a0cc0a64e5ad46fc87a9917cd760d0db] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=47.5 K 2024-11-14T21:33:15,581 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f52ab1047084866bdec9c9e73660826, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731619991291 2024-11-14T21:33:15,581 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 940f1587ea2c45baa6e87eae859bc80c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731619991752 2024-11-14T21:33:15,582 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf9824eaf74c4a37b336ab01fb73c67a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731619992942 2024-11-14T21:33:15,583 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0cc0a64e5ad46fc87a9917cd760d0db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731619994189 2024-11-14T21:33:15,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/50e3b94af4e64aa6aa6664d621b91cd9 is 50, key is test_row_0/B:col10/1731619994357/Put/seqid=0 2024-11-14T21:33:15,595 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#C#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:15,595 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/4d313a4dcf8d44e4892e0daca66db77b is 50, key is test_row_0/C:col10/1731619994193/Put/seqid=0 2024-11-14T21:33:15,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731619995576, max timestamp=1731619995594Num uniques 859; 2024-11-14T21:33:15,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731619995576, max timestamp=1731619995594Num uniques 859; 2024-11-14T21:33:15,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:33:15,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:33:15,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742077_1253 (size=12151) 2024-11-14T21:33:15,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/50e3b94af4e64aa6aa6664d621b91cd9 2024-11-14T21:33:15,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742078_1254 (size=12493) 2024-11-14T21:33:15,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620055623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:15,632 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/4d313a4dcf8d44e4892e0daca66db77b as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/4d313a4dcf8d44e4892e0daca66db77b 2024-11-14T21:33:15,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620055628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:15,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620055630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:15,644 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/C of 99a2a424d1830a6f519f1971fbc9bf41 into 4d313a4dcf8d44e4892e0daca66db77b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:15,644 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:15,644 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/C, priority=12, startTime=1731619995438; duration=0sec 2024-11-14T21:33:15,644 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:15,644 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:C 2024-11-14T21:33:15,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/7a283e0b2c5843b6a96beb6b55c80dec is 50, key is test_row_0/C:col10/1731619994357/Put/seqid=0 2024-11-14T21:33:15,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742079_1255 (size=12151) 2024-11-14T21:33:15,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/7a283e0b2c5843b6a96beb6b55c80dec 2024-11-14T21:33:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T21:33:15,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-14T21:33:15,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:15,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:15,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:15,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:15,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:15,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/4189428e0b8640bd811c9400e1d45ecc as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/4189428e0b8640bd811c9400e1d45ecc 2024-11-14T21:33:15,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/4189428e0b8640bd811c9400e1d45ecc, entries=200, sequenceid=195, filesize=14.2 K 2024-11-14T21:33:15,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/50e3b94af4e64aa6aa6664d621b91cd9 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/50e3b94af4e64aa6aa6664d621b91cd9 2024-11-14T21:33:15,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/50e3b94af4e64aa6aa6664d621b91cd9, entries=150, sequenceid=195, filesize=11.9 K 2024-11-14T21:33:15,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/7a283e0b2c5843b6a96beb6b55c80dec as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/7a283e0b2c5843b6a96beb6b55c80dec 2024-11-14T21:33:15,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:15,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620055740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:15,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:15,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620055740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:15,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:15,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620055742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:15,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/7a283e0b2c5843b6a96beb6b55c80dec, entries=150, sequenceid=195, filesize=11.9 K 2024-11-14T21:33:15,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=118.53 KB/121370 for 99a2a424d1830a6f519f1971fbc9bf41 in 241ms, sequenceid=195, compaction requested=false 2024-11-14T21:33:15,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:15,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-14T21:33:15,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:15,871 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=118.53 KB heapSize=343.47 KB 2024-11-14T21:33:15,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:15,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:15,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:15,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:15,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:15,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:15,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/6e35fe2a084a40cb8514d5e4a5db2839 is 50, key is test_row_0/A:col10/1731619995628/Put/seqid=0 2024-11-14T21:33:15,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742080_1256 (size=12151) 2024-11-14T21:33:15,944 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/6e35fe2a084a40cb8514d5e4a5db2839 2024-11-14T21:33:15,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:15,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:15,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/8e0302d68f4e4f2c820cab3e3beb03e4 is 50, key is test_row_0/B:col10/1731619995628/Put/seqid=0 2024-11-14T21:33:15,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620055985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:15,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620055987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:15,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:15,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:15,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:16,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:16,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620056000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:16,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:16,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46318 deadline: 1731620056003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:16,014 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_0' in 99a2a424d1830a6f519f1971fbc9bf41 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:33:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:16,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T21:33:16,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:16,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1731620056026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:16,035 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_1' in 99a2a424d1830a6f519f1971fbc9bf41 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:33:16,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:16,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742081_1257 (size=12151) 2024-11-14T21:33:16,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:16,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620056100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:16,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:16,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:16,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620056110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:16,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:16,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620056111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:16,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:16,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:16,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620056314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:16,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:16,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620056331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:16,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:16,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620056331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:16,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:16,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:16,467 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/8e0302d68f4e4f2c820cab3e3beb03e4 2024-11-14T21:33:16,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/bb2ed9522fd14047a77231eb156c856b is 50, key is test_row_0/C:col10/1731619995628/Put/seqid=0 2024-11-14T21:33:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T21:33:16,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742082_1258 (size=12151) 2024-11-14T21:33:16,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620056631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:16,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:16,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:16,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620056640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:16,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:16,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620056640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:16,932 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/bb2ed9522fd14047a77231eb156c856b 2024-11-14T21:33:16,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/6e35fe2a084a40cb8514d5e4a5db2839 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/6e35fe2a084a40cb8514d5e4a5db2839 2024-11-14T21:33:16,961 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/6e35fe2a084a40cb8514d5e4a5db2839, entries=150, sequenceid=228, filesize=11.9 K 2024-11-14T21:33:16,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/8e0302d68f4e4f2c820cab3e3beb03e4 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/8e0302d68f4e4f2c820cab3e3beb03e4 2024-11-14T21:33:16,973 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/8e0302d68f4e4f2c820cab3e3beb03e4, entries=150, sequenceid=228, filesize=11.9 K 2024-11-14T21:33:16,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/bb2ed9522fd14047a77231eb156c856b as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/bb2ed9522fd14047a77231eb156c856b 2024-11-14T21:33:16,983 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/bb2ed9522fd14047a77231eb156c856b, entries=150, sequenceid=228, filesize=11.9 K 2024-11-14T21:33:16,984 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~118.53 KB/121370, heapSize ~343.42 KB/351664, currentSize=60.38 KB/61830 for 99a2a424d1830a6f519f1971fbc9bf41 in 1113ms, sequenceid=228, compaction requested=true 2024-11-14T21:33:16,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:16,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:16,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-14T21:33:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-14T21:33:16,990 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-14T21:33:16,990 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5990 sec 2024-11-14T21:33:16,997 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.6080 sec 2024-11-14T21:33:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:17,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T21:33:17,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:17,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:17,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:17,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:17,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:17,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:17,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/bf9eeee2f25e4b5bbc95d7a2e0f52169 is 50, key is test_row_0/A:col10/1731619995987/Put/seqid=0 2024-11-14T21:33:17,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742083_1259 (size=12151) 2024-11-14T21:33:17,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/bf9eeee2f25e4b5bbc95d7a2e0f52169 2024-11-14T21:33:17,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/931647dd849144478dce9cbee885f627 is 50, key is test_row_0/B:col10/1731619995987/Put/seqid=0 2024-11-14T21:33:17,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742084_1260 (size=12151) 2024-11-14T21:33:17,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/931647dd849144478dce9cbee885f627 2024-11-14T21:33:17,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620057197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:17,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:17,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620057198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620057199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:17,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/03d6b5e435384ea283669e16544c9d8d is 50, key is test_row_0/C:col10/1731619995987/Put/seqid=0 2024-11-14T21:33:17,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742085_1261 (size=12151) 2024-11-14T21:33:17,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/03d6b5e435384ea283669e16544c9d8d 2024-11-14T21:33:17,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/bf9eeee2f25e4b5bbc95d7a2e0f52169 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/bf9eeee2f25e4b5bbc95d7a2e0f52169 2024-11-14T21:33:17,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/bf9eeee2f25e4b5bbc95d7a2e0f52169, entries=150, sequenceid=241, filesize=11.9 K 2024-11-14T21:33:17,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/931647dd849144478dce9cbee885f627 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/931647dd849144478dce9cbee885f627 2024-11-14T21:33:17,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/931647dd849144478dce9cbee885f627, entries=150, sequenceid=241, filesize=11.9 K 2024-11-14T21:33:17,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/03d6b5e435384ea283669e16544c9d8d as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/03d6b5e435384ea283669e16544c9d8d 2024-11-14T21:33:17,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/03d6b5e435384ea283669e16544c9d8d, entries=150, sequenceid=241, filesize=11.9 K 2024-11-14T21:33:17,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 99a2a424d1830a6f519f1971fbc9bf41 in 122ms, sequenceid=241, compaction requested=true 2024-11-14T21:33:17,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:17,265 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:33:17,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:33:17,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:17,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:33:17,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:17,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:33:17,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:17,266 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:33:17,270 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51336 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:33:17,270 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/A is initiating minor compaction (all files) 2024-11-14T21:33:17,270 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/A in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:17,270 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/b1a99692dd404dee8fcc23d49696e18c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/4189428e0b8640bd811c9400e1d45ecc, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/6e35fe2a084a40cb8514d5e4a5db2839, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/bf9eeee2f25e4b5bbc95d7a2e0f52169] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=50.1 K 2024-11-14T21:33:17,274 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1a99692dd404dee8fcc23d49696e18c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731619994189 2024-11-14T21:33:17,274 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:33:17,275 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4189428e0b8640bd811c9400e1d45ecc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731619994337 2024-11-14T21:33:17,275 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/B is initiating minor compaction (all files) 2024-11-14T21:33:17,275 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/B in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:17,275 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b0525bc3c0a04c9dbbf3f97ed624df46, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/50e3b94af4e64aa6aa6664d621b91cd9, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/8e0302d68f4e4f2c820cab3e3beb03e4, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/931647dd849144478dce9cbee885f627] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=47.8 K 2024-11-14T21:33:17,275 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e35fe2a084a40cb8514d5e4a5db2839, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731619995616 2024-11-14T21:33:17,275 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting b0525bc3c0a04c9dbbf3f97ed624df46, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731619994189 2024-11-14T21:33:17,275 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf9eeee2f25e4b5bbc95d7a2e0f52169, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731619995982 2024-11-14T21:33:17,276 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 50e3b94af4e64aa6aa6664d621b91cd9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731619994340 2024-11-14T21:33:17,276 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e0302d68f4e4f2c820cab3e3beb03e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731619995616 2024-11-14T21:33:17,277 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 931647dd849144478dce9cbee885f627, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731619995982 2024-11-14T21:33:17,296 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#A#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:17,296 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/f2ec75d36fc94014826a54710ef7635b is 50, key is test_row_0/A:col10/1731619995987/Put/seqid=0 2024-11-14T21:33:17,309 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#B#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:17,310 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/d85dd532cecd4992b0b76ec759bb933b is 50, key is test_row_0/B:col10/1731619995987/Put/seqid=0 2024-11-14T21:33:17,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:17,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-14T21:33:17,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:17,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:17,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:17,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:17,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:17,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:17,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/ca96259b7942495ab95620272138fd05 is 50, key is test_row_0/A:col10/1731619997183/Put/seqid=0 2024-11-14T21:33:17,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620057330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:17,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:17,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742087_1263 (size=12629) 2024-11-14T21:33:17,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620057335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:17,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:17,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620057337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:17,356 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/d85dd532cecd4992b0b76ec759bb933b as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/d85dd532cecd4992b0b76ec759bb933b 2024-11-14T21:33:17,366 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/B of 99a2a424d1830a6f519f1971fbc9bf41 into d85dd532cecd4992b0b76ec759bb933b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:17,366 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:17,366 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/B, priority=12, startTime=1731619997265; duration=0sec 2024-11-14T21:33:17,366 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:17,366 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:B 2024-11-14T21:33:17,366 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:33:17,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742088_1264 (size=14691) 2024-11-14T21:33:17,371 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:33:17,372 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/C is initiating minor compaction (all files) 2024-11-14T21:33:17,372 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/C in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:17,372 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/4d313a4dcf8d44e4892e0daca66db77b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/7a283e0b2c5843b6a96beb6b55c80dec, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/bb2ed9522fd14047a77231eb156c856b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/03d6b5e435384ea283669e16544c9d8d] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=47.8 K 2024-11-14T21:33:17,372 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d313a4dcf8d44e4892e0daca66db77b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731619994189 2024-11-14T21:33:17,373 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a283e0b2c5843b6a96beb6b55c80dec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731619994340 2024-11-14T21:33:17,374 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting bb2ed9522fd14047a77231eb156c856b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731619995616 2024-11-14T21:33:17,374 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 03d6b5e435384ea283669e16544c9d8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731619995982 2024-11-14T21:33:17,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742086_1262 (size=12629) 2024-11-14T21:33:17,389 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#C#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:17,390 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/d4bc0c49e3eb45c3a819d129c7a06bbb is 50, key is test_row_0/C:col10/1731619995987/Put/seqid=0 2024-11-14T21:33:17,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742089_1265 (size=12629) 2024-11-14T21:33:17,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:17,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620057440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:17,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:17,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:17,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620057450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:17,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620057450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:17,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T21:33:17,542 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:33:17,542 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:33:17,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-14T21:33:17,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T21:33:17,548 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:33:17,549 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:33:17,549 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:33:17,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:17,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620057651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:17,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:17,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T21:33:17,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:17,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620057662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:17,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:17,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620057662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:17,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T21:33:17,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:17,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:17,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:17,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:17,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:17,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:17,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/ca96259b7942495ab95620272138fd05 2024-11-14T21:33:17,784 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/f2ec75d36fc94014826a54710ef7635b as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/f2ec75d36fc94014826a54710ef7635b 2024-11-14T21:33:17,792 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/A of 99a2a424d1830a6f519f1971fbc9bf41 into f2ec75d36fc94014826a54710ef7635b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:17,792 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:17,792 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/A, priority=12, startTime=1731619997265; duration=0sec 2024-11-14T21:33:17,792 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:17,792 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:A 2024-11-14T21:33:17,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/11ed7269b6a443d8a64a79af830a97b7 is 50, key is test_row_0/B:col10/1731619997183/Put/seqid=0 2024-11-14T21:33:17,822 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/d4bc0c49e3eb45c3a819d129c7a06bbb as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/d4bc0c49e3eb45c3a819d129c7a06bbb 2024-11-14T21:33:17,828 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/C of 99a2a424d1830a6f519f1971fbc9bf41 into d4bc0c49e3eb45c3a819d129c7a06bbb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:17,828 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:17,828 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/C, priority=12, startTime=1731619997265; duration=0sec 2024-11-14T21:33:17,828 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:17,828 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:C 2024-11-14T21:33:17,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742090_1266 (size=12251) 2024-11-14T21:33:17,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/11ed7269b6a443d8a64a79af830a97b7 2024-11-14T21:33:17,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T21:33:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T21:33:17,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:17,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:17,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:17,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:17,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:17,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:17,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/b532eef1b28741de98f1041d79a793c2 is 50, key is test_row_0/C:col10/1731619997183/Put/seqid=0 2024-11-14T21:33:17,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742091_1267 (size=12251) 2024-11-14T21:33:17,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620057962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:17,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:17,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620057970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:17,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620057971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:17,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:18,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T21:33:18,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:18,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:18,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:18,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:18,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:18,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T21:33:18,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:18,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:18,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:18,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:18,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T21:33:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:18,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/b532eef1b28741de98f1041d79a793c2 2024-11-14T21:33:18,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T21:33:18,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:18,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:18,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:18,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:18,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/ca96259b7942495ab95620272138fd05 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/ca96259b7942495ab95620272138fd05 2024-11-14T21:33:18,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/ca96259b7942495ab95620272138fd05, entries=200, sequenceid=262, filesize=14.3 K 2024-11-14T21:33:18,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/11ed7269b6a443d8a64a79af830a97b7 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/11ed7269b6a443d8a64a79af830a97b7 2024-11-14T21:33:18,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/11ed7269b6a443d8a64a79af830a97b7, entries=150, sequenceid=262, filesize=12.0 K 2024-11-14T21:33:18,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/b532eef1b28741de98f1041d79a793c2 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/b532eef1b28741de98f1041d79a793c2 2024-11-14T21:33:18,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/b532eef1b28741de98f1041d79a793c2, entries=150, sequenceid=262, filesize=12.0 K 2024-11-14T21:33:18,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 99a2a424d1830a6f519f1971fbc9bf41 in 1067ms, sequenceid=262, compaction requested=false 2024-11-14T21:33:18,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:18,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T21:33:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:18,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:18,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:18,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:18,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:18,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:18,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:18,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T21:33:18,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:18,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:18,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:18,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/e455527c6763454293b4b3051c9d548c is 50, key is test_row_0/A:col10/1731619998486/Put/seqid=0 2024-11-14T21:33:18,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:18,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:18,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742092_1268 (size=12301) 2024-11-14T21:33:18,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/e455527c6763454293b4b3051c9d548c 2024-11-14T21:33:18,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/efa5253aca3b4f55b89845017ed903ed is 50, key is test_row_0/B:col10/1731619998486/Put/seqid=0 2024-11-14T21:33:18,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620058576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:18,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620058583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:18,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620058584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:18,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742093_1269 (size=12301) 2024-11-14T21:33:18,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/efa5253aca3b4f55b89845017ed903ed 2024-11-14T21:33:18,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/c8789e64f67a48ecbff66cd2cb0c37ee is 50, key is test_row_0/C:col10/1731619998486/Put/seqid=0 2024-11-14T21:33:18,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742094_1270 (size=12301) 2024-11-14T21:33:18,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/c8789e64f67a48ecbff66cd2cb0c37ee 2024-11-14T21:33:18,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/e455527c6763454293b4b3051c9d548c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/e455527c6763454293b4b3051c9d548c 2024-11-14T21:33:18,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T21:33:18,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:18,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:18,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:18,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:18,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:18,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/e455527c6763454293b4b3051c9d548c, entries=150, sequenceid=278, filesize=12.0 K 2024-11-14T21:33:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T21:33:18,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620058691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:18,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/efa5253aca3b4f55b89845017ed903ed as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/efa5253aca3b4f55b89845017ed903ed 2024-11-14T21:33:18,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:18,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:18,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:18,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/efa5253aca3b4f55b89845017ed903ed, entries=150, sequenceid=278, filesize=12.0 K 2024-11-14T21:33:18,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/c8789e64f67a48ecbff66cd2cb0c37ee as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/c8789e64f67a48ecbff66cd2cb0c37ee 2024-11-14T21:33:18,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/c8789e64f67a48ecbff66cd2cb0c37ee, entries=150, sequenceid=278, filesize=12.0 K 2024-11-14T21:33:18,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 99a2a424d1830a6f519f1971fbc9bf41 in 220ms, sequenceid=278, compaction requested=true 2024-11-14T21:33:18,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:18,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:33:18,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:18,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:33:18,709 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:18,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:18,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:33:18,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:33:18,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-14T21:33:18,709 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:18,710 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39621 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:18,711 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/A is initiating minor compaction (all files) 2024-11-14T21:33:18,711 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/A in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:18,711 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/f2ec75d36fc94014826a54710ef7635b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/ca96259b7942495ab95620272138fd05, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/e455527c6763454293b4b3051c9d548c] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=38.7 K 2024-11-14T21:33:18,711 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37181 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:18,711 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/B is initiating minor compaction (all files) 2024-11-14T21:33:18,711 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/B in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:18,711 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/d85dd532cecd4992b0b76ec759bb933b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/11ed7269b6a443d8a64a79af830a97b7, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/efa5253aca3b4f55b89845017ed903ed] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=36.3 K 2024-11-14T21:33:18,712 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2ec75d36fc94014826a54710ef7635b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731619995982 2024-11-14T21:33:18,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731619998555, max timestamp=1731619998707Num uniques 859; 2024-11-14T21:33:18,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:33:18,714 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting d85dd532cecd4992b0b76ec759bb933b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731619995982 2024-11-14T21:33:18,714 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca96259b7942495ab95620272138fd05, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731619997183 2024-11-14T21:33:18,714 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 11ed7269b6a443d8a64a79af830a97b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731619997183 2024-11-14T21:33:18,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:18,715 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting e455527c6763454293b4b3051c9d548c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731619997335 2024-11-14T21:33:18,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:18,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:18,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:18,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:18,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:18,716 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting efa5253aca3b4f55b89845017ed903ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731619997335 2024-11-14T21:33:18,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/c51f23a7dc444a7d8f1587d2db1c55ab is 50, key is test_row_0/A:col10/1731619998578/Put/seqid=0 2024-11-14T21:33:18,734 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#B#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:18,735 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/941298d6df984aec8602b86d9d729875 is 50, key is test_row_0/B:col10/1731619998486/Put/seqid=0 2024-11-14T21:33:18,773 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#A#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:18,774 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/8b56b496b9f34a45a4d65f7739874a0d is 50, key is test_row_0/A:col10/1731619998486/Put/seqid=0 2024-11-14T21:33:18,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742095_1271 (size=14741) 2024-11-14T21:33:18,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620058780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:18,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620058782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:18,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:18,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:18,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:18,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742096_1272 (size=12881) 2024-11-14T21:33:18,809 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/941298d6df984aec8602b86d9d729875 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/941298d6df984aec8602b86d9d729875 2024-11-14T21:33:18,816 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/B of 99a2a424d1830a6f519f1971fbc9bf41 into 941298d6df984aec8602b86d9d729875(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:18,816 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:18,816 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/B, priority=13, startTime=1731619998709; duration=0sec 2024-11-14T21:33:18,816 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:18,816 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:B 2024-11-14T21:33:18,816 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:18,819 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37181 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:18,819 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/C is initiating minor compaction (all files) 2024-11-14T21:33:18,819 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/C in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:18,819 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/d4bc0c49e3eb45c3a819d129c7a06bbb, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/b532eef1b28741de98f1041d79a793c2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/c8789e64f67a48ecbff66cd2cb0c37ee] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=36.3 K 2024-11-14T21:33:18,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742097_1273 (size=12881) 2024-11-14T21:33:18,821 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting d4bc0c49e3eb45c3a819d129c7a06bbb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731619995982 2024-11-14T21:33:18,821 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting b532eef1b28741de98f1041d79a793c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731619997183 2024-11-14T21:33:18,822 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting c8789e64f67a48ecbff66cd2cb0c37ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731619997335 2024-11-14T21:33:18,833 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/8b56b496b9f34a45a4d65f7739874a0d as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/8b56b496b9f34a45a4d65f7739874a0d 2024-11-14T21:33:18,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T21:33:18,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:18,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:18,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:18,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:18,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:18,840 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/A of 99a2a424d1830a6f519f1971fbc9bf41 into 8b56b496b9f34a45a4d65f7739874a0d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:18,840 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:18,840 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/A, priority=13, startTime=1731619998709; duration=0sec 2024-11-14T21:33:18,840 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:18,840 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:A 2024-11-14T21:33:18,844 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#C#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:18,845 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/5ae9bd241e8a432bac30ad1f70397237 is 50, key is test_row_0/C:col10/1731619998486/Put/seqid=0 2024-11-14T21:33:18,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742098_1274 (size=12881) 2024-11-14T21:33:18,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620058903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:18,910 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/5ae9bd241e8a432bac30ad1f70397237 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/5ae9bd241e8a432bac30ad1f70397237 2024-11-14T21:33:18,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:18,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620058903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620058903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:18,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:18,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:18,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:18,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:18,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:18,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:18,938 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/C of 99a2a424d1830a6f519f1971fbc9bf41 into 5ae9bd241e8a432bac30ad1f70397237(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:18,938 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:18,938 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/C, priority=13, startTime=1731619998709; duration=0sec 2024-11-14T21:33:18,938 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:18,938 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:C 2024-11-14T21:33:18,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T21:33:18,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:18,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:19,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:19,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:19,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:19,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620059120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:19,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620059122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:19,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:19,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:19,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:19,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T21:33:19,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:19,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:19,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:19,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:19,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:19,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/c51f23a7dc444a7d8f1587d2db1c55ab 2024-11-14T21:33:19,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/9e160f4a03164f8986e575bc12c0d6a1 is 50, key is test_row_0/B:col10/1731619998578/Put/seqid=0 2024-11-14T21:33:19,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742099_1275 (size=12301) 2024-11-14T21:33:19,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:19,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620059221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:19,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T21:33:19,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:19,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:19,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:19,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:19,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:19,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:19,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620059431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:19,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:19,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:19,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:19,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620059440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:19,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T21:33:19,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:19,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:19,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:19,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:19,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:19,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:19,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T21:33:19,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:19,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:19,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:19,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:19,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:19,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:19,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/9e160f4a03164f8986e575bc12c0d6a1 2024-11-14T21:33:19,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/47b27354a50145d0b7a7b3451b9e3fe0 is 50, key is test_row_0/C:col10/1731619998578/Put/seqid=0 2024-11-14T21:33:19,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742100_1276 (size=12301) 2024-11-14T21:33:19,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T21:33:19,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620059740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:19,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:19,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:19,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:19,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T21:33:19,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:19,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:19,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:19,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:19,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:19,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:19,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T21:33:19,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:19,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:19,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:19,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:19,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:19,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620059941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:19,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620059951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:19,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:19,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:19,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:20,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:20,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46318 deadline: 1731620060040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:20,045 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_0' in 99a2a424d1830a6f519f1971fbc9bf41 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8268 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:33:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:20,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/47b27354a50145d0b7a7b3451b9e3fe0 2024-11-14T21:33:20,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/c51f23a7dc444a7d8f1587d2db1c55ab as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/c51f23a7dc444a7d8f1587d2db1c55ab 2024-11-14T21:33:20,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/c51f23a7dc444a7d8f1587d2db1c55ab, entries=200, sequenceid=300, filesize=14.4 K 2024-11-14T21:33:20,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/9e160f4a03164f8986e575bc12c0d6a1 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/9e160f4a03164f8986e575bc12c0d6a1 2024-11-14T21:33:20,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/9e160f4a03164f8986e575bc12c0d6a1, entries=150, sequenceid=300, filesize=12.0 K 2024-11-14T21:33:20,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/47b27354a50145d0b7a7b3451b9e3fe0 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/47b27354a50145d0b7a7b3451b9e3fe0 2024-11-14T21:33:20,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1731620060070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:20,078 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_1' in 99a2a424d1830a6f519f1971fbc9bf41 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8303 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:33:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:20,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/47b27354a50145d0b7a7b3451b9e3fe0, entries=150, sequenceid=300, filesize=12.0 K 2024-11-14T21:33:20,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~96.16 KB/98470, heapSize ~281.98 KB/288752, currentSize=80.51 KB/82440 for 99a2a424d1830a6f519f1971fbc9bf41 in 1375ms, sequenceid=300, compaction requested=false 2024-11-14T21:33:20,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:20,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T21:33:20,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:20,087 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T21:33:20,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:20,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:20,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:20,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:20,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:20,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:20,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/74848ce380c141e88cdc35c02fcdd197 is 50, key is test_row_0/A:col10/1731619998780/Put/seqid=0 2024-11-14T21:33:20,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742101_1277 (size=12301) 2024-11-14T21:33:20,575 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/74848ce380c141e88cdc35c02fcdd197 2024-11-14T21:33:20,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/7524c6d199584bd8b78b776630019d10 is 50, key is test_row_0/B:col10/1731619998780/Put/seqid=0 2024-11-14T21:33:20,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742102_1278 (size=12301) 2024-11-14T21:33:20,702 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/7524c6d199584bd8b78b776630019d10 2024-11-14T21:33:20,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/48d53852b6144a3c8b46e8e483223c29 is 50, key is test_row_0/C:col10/1731619998780/Put/seqid=0 2024-11-14T21:33:20,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:20,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:20,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742103_1279 (size=12301) 2024-11-14T21:33:20,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620060925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:20,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620060950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:20,954 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_2' in 99a2a424d1830a6f519f1971fbc9bf41 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:33:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:20,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620060960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:20,965 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_0' in 99a2a424d1830a6f519f1971fbc9bf41 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:33:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:21,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620061042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:21,200 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/48d53852b6144a3c8b46e8e483223c29 2024-11-14T21:33:21,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/74848ce380c141e88cdc35c02fcdd197 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/74848ce380c141e88cdc35c02fcdd197 2024-11-14T21:33:21,220 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/74848ce380c141e88cdc35c02fcdd197, entries=150, sequenceid=318, filesize=12.0 K 2024-11-14T21:33:21,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/7524c6d199584bd8b78b776630019d10 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/7524c6d199584bd8b78b776630019d10 2024-11-14T21:33:21,231 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/7524c6d199584bd8b78b776630019d10, entries=150, sequenceid=318, filesize=12.0 K 2024-11-14T21:33:21,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/48d53852b6144a3c8b46e8e483223c29 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/48d53852b6144a3c8b46e8e483223c29 2024-11-14T21:33:21,236 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/48d53852b6144a3c8b46e8e483223c29, entries=150, sequenceid=318, filesize=12.0 K 2024-11-14T21:33:21,237 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 99a2a424d1830a6f519f1971fbc9bf41 in 1150ms, sequenceid=318, compaction requested=true 2024-11-14T21:33:21,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:21,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:21,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-14T21:33:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-14T21:33:21,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-14T21:33:21,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6900 sec 2024-11-14T21:33:21,245 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 3.7020 sec 2024-11-14T21:33:21,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:21,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T21:33:21,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:21,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:21,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:21,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:21,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:21,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:21,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/155c50021693490fa8435e597a839640 is 50, key is test_row_0/A:col10/1731620000914/Put/seqid=0 2024-11-14T21:33:21,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742104_1280 (size=14741) 2024-11-14T21:33:21,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:21,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620061309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:21,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:21,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:21,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:21,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620061422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:21,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620061641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:21,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/155c50021693490fa8435e597a839640 2024-11-14T21:33:21,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/b27d7f44dfab4983a52b6bbb61abb83c is 50, key is test_row_0/B:col10/1731620000914/Put/seqid=0 2024-11-14T21:33:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T21:33:21,727 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:33:21,730 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:33:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-14T21:33:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T21:33:21,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742105_1281 (size=12301) 2024-11-14T21:33:21,748 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:33:21,750 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:33:21,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/b27d7f44dfab4983a52b6bbb61abb83c 2024-11-14T21:33:21,750 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:33:21,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/0326ff60302245f899ac4f730dffbb4c is 50, key is test_row_0/C:col10/1731620000914/Put/seqid=0 2024-11-14T21:33:21,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742106_1282 (size=12301) 2024-11-14T21:33:21,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/0326ff60302245f899ac4f730dffbb4c 2024-11-14T21:33:21,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/155c50021693490fa8435e597a839640 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/155c50021693490fa8435e597a839640 2024-11-14T21:33:21,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/155c50021693490fa8435e597a839640, entries=200, sequenceid=336, filesize=14.4 K 2024-11-14T21:33:21,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/b27d7f44dfab4983a52b6bbb61abb83c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b27d7f44dfab4983a52b6bbb61abb83c 2024-11-14T21:33:21,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b27d7f44dfab4983a52b6bbb61abb83c, entries=150, sequenceid=336, filesize=12.0 K 2024-11-14T21:33:21,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/0326ff60302245f899ac4f730dffbb4c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/0326ff60302245f899ac4f730dffbb4c 2024-11-14T21:33:21,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/0326ff60302245f899ac4f730dffbb4c, entries=150, sequenceid=336, filesize=12.0 K 2024-11-14T21:33:21,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 99a2a424d1830a6f519f1971fbc9bf41 in 595ms, sequenceid=336, compaction requested=true 2024-11-14T21:33:21,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T21:33:21,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:33:21,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:21,850 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:33:21,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:33:21,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:21,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:33:21,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:21,851 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:33:21,852 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54664 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:33:21,852 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/A is initiating minor compaction (all files) 2024-11-14T21:33:21,852 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/A in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:21,852 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/8b56b496b9f34a45a4d65f7739874a0d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/c51f23a7dc444a7d8f1587d2db1c55ab, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/74848ce380c141e88cdc35c02fcdd197, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/155c50021693490fa8435e597a839640] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=53.4 K 2024-11-14T21:33:21,853 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b56b496b9f34a45a4d65f7739874a0d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731619997335 2024-11-14T21:33:21,854 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting c51f23a7dc444a7d8f1587d2db1c55ab, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731619998555 2024-11-14T21:33:21,854 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74848ce380c141e88cdc35c02fcdd197, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731619998755 2024-11-14T21:33:21,854 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:33:21,854 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/B is initiating minor compaction (all files) 2024-11-14T21:33:21,854 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/B in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:21,854 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/941298d6df984aec8602b86d9d729875, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/9e160f4a03164f8986e575bc12c0d6a1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/7524c6d199584bd8b78b776630019d10, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b27d7f44dfab4983a52b6bbb61abb83c] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=48.6 K 2024-11-14T21:33:21,855 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 155c50021693490fa8435e597a839640, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731620000887 2024-11-14T21:33:21,855 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 941298d6df984aec8602b86d9d729875, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731619997335 2024-11-14T21:33:21,855 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e160f4a03164f8986e575bc12c0d6a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731619998578 2024-11-14T21:33:21,862 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 7524c6d199584bd8b78b776630019d10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731619998755 2024-11-14T21:33:21,863 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting b27d7f44dfab4983a52b6bbb61abb83c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731620000914 2024-11-14T21:33:21,885 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#A#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:21,905 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/f5644b35cd624c039abc894aba0ec960 is 50, key is test_row_0/A:col10/1731620000914/Put/seqid=0 2024-11-14T21:33:21,908 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#B#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:21,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T21:33:21,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:21,909 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T21:33:21,909 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/abcbfc8d90824778a6105289487fb174 is 50, key is test_row_0/B:col10/1731620000914/Put/seqid=0 2024-11-14T21:33:21,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:21,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:21,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:21,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:21,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:21,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:21,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/90b0d5506b6a4a88b76c6f0935765dd4 is 50, key is test_row_0/A:col10/1731620001307/Put/seqid=0 2024-11-14T21:33:21,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742108_1284 (size=13017) 2024-11-14T21:33:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:21,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:21,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742107_1283 (size=13017) 2024-11-14T21:33:21,958 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/abcbfc8d90824778a6105289487fb174 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/abcbfc8d90824778a6105289487fb174 2024-11-14T21:33:21,967 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/f5644b35cd624c039abc894aba0ec960 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/f5644b35cd624c039abc894aba0ec960 2024-11-14T21:33:21,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742109_1285 (size=12301) 2024-11-14T21:33:21,974 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/B of 99a2a424d1830a6f519f1971fbc9bf41 into abcbfc8d90824778a6105289487fb174(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:21,974 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:21,974 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/B, priority=12, startTime=1731620001850; duration=0sec 2024-11-14T21:33:21,974 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:21,974 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:B 2024-11-14T21:33:21,974 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:33:21,979 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/90b0d5506b6a4a88b76c6f0935765dd4 2024-11-14T21:33:21,980 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:33:21,980 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/C is initiating minor compaction (all files) 2024-11-14T21:33:21,980 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/C in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:21,980 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/5ae9bd241e8a432bac30ad1f70397237, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/47b27354a50145d0b7a7b3451b9e3fe0, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/48d53852b6144a3c8b46e8e483223c29, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/0326ff60302245f899ac4f730dffbb4c] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=48.6 K 2024-11-14T21:33:21,982 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ae9bd241e8a432bac30ad1f70397237, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731619997335 2024-11-14T21:33:21,982 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 47b27354a50145d0b7a7b3451b9e3fe0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731619998578 2024-11-14T21:33:21,983 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 48d53852b6144a3c8b46e8e483223c29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731619998755 2024-11-14T21:33:21,983 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 0326ff60302245f899ac4f730dffbb4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731620000914 2024-11-14T21:33:21,986 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/A of 99a2a424d1830a6f519f1971fbc9bf41 into f5644b35cd624c039abc894aba0ec960(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:21,986 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:21,987 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/A, priority=12, startTime=1731620001850; duration=0sec 2024-11-14T21:33:21,987 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:21,987 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:A 2024-11-14T21:33:21,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/b743cbc2fbe54072bbec5734f9785061 is 50, key is test_row_0/B:col10/1731620001307/Put/seqid=0 2024-11-14T21:33:22,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742110_1286 (size=12301) 2024-11-14T21:33:22,004 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#C#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:22,005 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/a58d9052cbc141dd88a8a987f9ed269c is 50, key is test_row_0/C:col10/1731620000914/Put/seqid=0 2024-11-14T21:33:22,006 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/b743cbc2fbe54072bbec5734f9785061 2024-11-14T21:33:22,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/fabf0b3704b2411a86309a39862b0594 is 50, key is test_row_0/C:col10/1731620001307/Put/seqid=0 2024-11-14T21:33:22,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742111_1287 (size=13017) 2024-11-14T21:33:22,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742112_1288 (size=12301) 2024-11-14T21:33:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T21:33:22,065 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/fabf0b3704b2411a86309a39862b0594 2024-11-14T21:33:22,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620062074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:22,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/90b0d5506b6a4a88b76c6f0935765dd4 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/90b0d5506b6a4a88b76c6f0935765dd4 2024-11-14T21:33:22,112 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/90b0d5506b6a4a88b76c6f0935765dd4, entries=150, sequenceid=350, filesize=12.0 K 2024-11-14T21:33:22,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/b743cbc2fbe54072bbec5734f9785061 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b743cbc2fbe54072bbec5734f9785061 2024-11-14T21:33:22,135 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b743cbc2fbe54072bbec5734f9785061, entries=150, sequenceid=350, filesize=12.0 K 2024-11-14T21:33:22,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/fabf0b3704b2411a86309a39862b0594 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/fabf0b3704b2411a86309a39862b0594 2024-11-14T21:33:22,172 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/fabf0b3704b2411a86309a39862b0594, entries=150, sequenceid=350, filesize=12.0 K 2024-11-14T21:33:22,174 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 99a2a424d1830a6f519f1971fbc9bf41 in 264ms, sequenceid=350, compaction requested=false 2024-11-14T21:33:22,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:22,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:22,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-14T21:33:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-14T21:33:22,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T21:33:22,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:22,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:22,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:22,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:22,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:22,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:22,214 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-14T21:33:22,215 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 436 msec 2024-11-14T21:33:22,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 492 msec 2024-11-14T21:33:22,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/0645f15a4a184ba29aed7edf65786005 is 50, key is test_row_0/A:col10/1731620002072/Put/seqid=0 2024-11-14T21:33:22,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620062280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:22,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742113_1289 (size=12301) 2024-11-14T21:33:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:22,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/0645f15a4a184ba29aed7edf65786005 2024-11-14T21:33:22,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/b2e095b5775442cf899f267528918010 is 50, key is test_row_0/B:col10/1731620002072/Put/seqid=0 2024-11-14T21:33:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T21:33:22,371 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:33:22,371 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:33:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-14T21:33:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T21:33:22,374 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:33:22,375 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:33:22,375 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:33:22,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742114_1290 (size=12301) 2024-11-14T21:33:22,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620062400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:22,435 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/a58d9052cbc141dd88a8a987f9ed269c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/a58d9052cbc141dd88a8a987f9ed269c 2024-11-14T21:33:22,452 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/C of 99a2a424d1830a6f519f1971fbc9bf41 into a58d9052cbc141dd88a8a987f9ed269c(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:22,452 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:22,452 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/C, priority=12, startTime=1731620001850; duration=0sec 2024-11-14T21:33:22,452 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:22,452 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:C 2024-11-14T21:33:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T21:33:22,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T21:33:22,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:22,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:22,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:22,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:22,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:22,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620062611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:22,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:22,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:22,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T21:33:22,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T21:33:22,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:22,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:22,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:22,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:22,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:22,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/b2e095b5775442cf899f267528918010 2024-11-14T21:33:22,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/09fe2b423a9d4d00aff4a8a7cc850a1a is 50, key is test_row_0/C:col10/1731620002072/Put/seqid=0 2024-11-14T21:33:22,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742115_1291 (size=12301) 2024-11-14T21:33:22,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T21:33:22,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:22,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:22,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:22,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:22,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:22,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620062920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:22,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:22,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:22,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620062971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:22,976 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_0' in 99a2a424d1830a6f519f1971fbc9bf41 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:33:22,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:22,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:22,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:22,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620062980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:22,984 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_2' in 99a2a424d1830a6f519f1971fbc9bf41 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:33:22,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:22,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:22,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T21:33:23,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T21:33:23,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:23,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:23,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:23,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:23,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:23,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T21:33:23,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:23,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:23,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:23,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:23,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:23,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/09fe2b423a9d4d00aff4a8a7cc850a1a 2024-11-14T21:33:23,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/0645f15a4a184ba29aed7edf65786005 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/0645f15a4a184ba29aed7edf65786005 2024-11-14T21:33:23,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/0645f15a4a184ba29aed7edf65786005, entries=150, sequenceid=371, filesize=12.0 K 2024-11-14T21:33:23,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/b2e095b5775442cf899f267528918010 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b2e095b5775442cf899f267528918010 2024-11-14T21:33:23,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b2e095b5775442cf899f267528918010, entries=150, sequenceid=371, filesize=12.0 K 2024-11-14T21:33:23,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/09fe2b423a9d4d00aff4a8a7cc850a1a as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/09fe2b423a9d4d00aff4a8a7cc850a1a 2024-11-14T21:33:23,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/09fe2b423a9d4d00aff4a8a7cc850a1a, entries=150, sequenceid=371, filesize=12.0 K 2024-11-14T21:33:23,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 99a2a424d1830a6f519f1971fbc9bf41 in 1100ms, sequenceid=371, compaction requested=true 2024-11-14T21:33:23,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:23,300 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:23,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:33:23,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:23,300 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:23,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:33:23,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:23,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:33:23,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:23,303 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:23,303 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/A is initiating minor compaction (all files) 2024-11-14T21:33:23,303 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/A in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:23,303 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/f5644b35cd624c039abc894aba0ec960, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/90b0d5506b6a4a88b76c6f0935765dd4, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/0645f15a4a184ba29aed7edf65786005] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=36.7 K 2024-11-14T21:33:23,303 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:23,303 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/B is initiating minor compaction (all files) 2024-11-14T21:33:23,303 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/B in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:23,303 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/abcbfc8d90824778a6105289487fb174, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b743cbc2fbe54072bbec5734f9785061, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b2e095b5775442cf899f267528918010] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=36.7 K 2024-11-14T21:33:23,305 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting abcbfc8d90824778a6105289487fb174, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731620000914 2024-11-14T21:33:23,305 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5644b35cd624c039abc894aba0ec960, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731620000914 2024-11-14T21:33:23,307 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting b743cbc2fbe54072bbec5734f9785061, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731620001277 2024-11-14T21:33:23,307 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90b0d5506b6a4a88b76c6f0935765dd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731620001277 2024-11-14T21:33:23,308 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting b2e095b5775442cf899f267528918010, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731620002057 2024-11-14T21:33:23,308 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0645f15a4a184ba29aed7edf65786005, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731620002057 2024-11-14T21:33:23,345 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#A#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:23,345 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/dd8db7f9791446dd9ca9dd83c6da001b is 50, key is test_row_0/A:col10/1731620002072/Put/seqid=0 2024-11-14T21:33:23,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742116_1292 (size=13119) 2024-11-14T21:33:23,359 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#B#compaction#252 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:23,359 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/febded1128e54857ac0aaf372997d71b is 50, key is test_row_0/B:col10/1731620002072/Put/seqid=0 2024-11-14T21:33:23,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T21:33:23,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:23,365 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T21:33:23,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:23,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:23,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:23,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:23,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:23,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:23,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/d5a9577e3f2649f98969abdcf098a49f is 50, key is test_row_0/A:col10/1731620002276/Put/seqid=0 2024-11-14T21:33:23,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742117_1293 (size=13119) 2024-11-14T21:33:23,420 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/febded1128e54857ac0aaf372997d71b as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/febded1128e54857ac0aaf372997d71b 2024-11-14T21:33:23,436 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/B of 99a2a424d1830a6f519f1971fbc9bf41 into febded1128e54857ac0aaf372997d71b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:23,436 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:23,436 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/B, priority=13, startTime=1731620003300; duration=0sec 2024-11-14T21:33:23,436 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:23,436 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:B 2024-11-14T21:33:23,436 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:23,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:23,451 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:23,451 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/C is initiating minor compaction (all files) 2024-11-14T21:33:23,451 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/C in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:23,451 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/a58d9052cbc141dd88a8a987f9ed269c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/fabf0b3704b2411a86309a39862b0594, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/09fe2b423a9d4d00aff4a8a7cc850a1a] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=36.7 K 2024-11-14T21:33:23,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742118_1294 (size=12301) 2024-11-14T21:33:23,457 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting a58d9052cbc141dd88a8a987f9ed269c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731620000914 2024-11-14T21:33:23,458 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting fabf0b3704b2411a86309a39862b0594, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731620001277 2024-11-14T21:33:23,459 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 09fe2b423a9d4d00aff4a8a7cc850a1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731620002057 2024-11-14T21:33:23,470 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#C#compaction#254 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:23,470 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/d396a17479df44f7ba38ec47cf423341 is 50, key is test_row_0/C:col10/1731620002072/Put/seqid=0 2024-11-14T21:33:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T21:33:23,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742119_1295 (size=13119) 2024-11-14T21:33:23,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620063632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:23,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:23,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:23,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:23,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:23,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620063741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:23,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:23,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:23,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:23,782 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/dd8db7f9791446dd9ca9dd83c6da001b as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/dd8db7f9791446dd9ca9dd83c6da001b 2024-11-14T21:33:23,814 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/A of 99a2a424d1830a6f519f1971fbc9bf41 into dd8db7f9791446dd9ca9dd83c6da001b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:23,815 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:23,815 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/A, priority=13, startTime=1731620003300; duration=0sec 2024-11-14T21:33:23,815 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:23,815 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:A 2024-11-14T21:33:23,858 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/d5a9577e3f2649f98969abdcf098a49f 2024-11-14T21:33:23,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/cd7dd8d49ff34e1c9e1d8ea3a58ac39f is 50, key is test_row_0/B:col10/1731620002276/Put/seqid=0 2024-11-14T21:33:23,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742120_1296 (size=12301) 2024-11-14T21:33:23,930 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/cd7dd8d49ff34e1c9e1d8ea3a58ac39f 2024-11-14T21:33:23,940 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/d396a17479df44f7ba38ec47cf423341 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/d396a17479df44f7ba38ec47cf423341 2024-11-14T21:33:23,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620063950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:23,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:23,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:23,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:23,959 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/C of 99a2a424d1830a6f519f1971fbc9bf41 into d396a17479df44f7ba38ec47cf423341(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:23,959 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:23,959 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/C, priority=13, startTime=1731620003300; duration=0sec 2024-11-14T21:33:23,959 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:23,959 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:C 2024-11-14T21:33:23,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/96b5bbf371ce433f90df612e3a9966da is 50, key is test_row_0/C:col10/1731620002276/Put/seqid=0 2024-11-14T21:33:24,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742121_1297 (size=12301) 2024-11-14T21:33:24,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620064262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:24,425 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/96b5bbf371ce433f90df612e3a9966da 2024-11-14T21:33:24,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/d5a9577e3f2649f98969abdcf098a49f as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/d5a9577e3f2649f98969abdcf098a49f 2024-11-14T21:33:24,519 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/d5a9577e3f2649f98969abdcf098a49f, entries=150, sequenceid=385, filesize=12.0 K 2024-11-14T21:33:24,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/cd7dd8d49ff34e1c9e1d8ea3a58ac39f as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/cd7dd8d49ff34e1c9e1d8ea3a58ac39f 2024-11-14T21:33:24,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T21:33:24,534 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/cd7dd8d49ff34e1c9e1d8ea3a58ac39f, entries=150, sequenceid=385, filesize=12.0 K 2024-11-14T21:33:24,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/96b5bbf371ce433f90df612e3a9966da as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/96b5bbf371ce433f90df612e3a9966da 2024-11-14T21:33:24,568 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/96b5bbf371ce433f90df612e3a9966da, entries=150, sequenceid=385, filesize=12.0 K 2024-11-14T21:33:24,569 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 99a2a424d1830a6f519f1971fbc9bf41 in 1204ms, sequenceid=385, compaction requested=false 2024-11-14T21:33:24,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:24,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:24,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-14T21:33:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-14T21:33:24,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-14T21:33:24,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1970 sec 2024-11-14T21:33:24,577 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.2040 sec 2024-11-14T21:33:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:24,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T21:33:24,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:24,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:24,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:24,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:24,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:24,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:24,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/a1c1fd42cfcc45a3b6d6b292b4b00186 is 50, key is test_row_0/A:col10/1731620003630/Put/seqid=0 2024-11-14T21:33:24,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742122_1298 (size=12301) 2024-11-14T21:33:24,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620064875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:25,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:25,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620065003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:25,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620065218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:25,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:25,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:25,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:25,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/a1c1fd42cfcc45a3b6d6b292b4b00186 2024-11-14T21:33:25,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/5ad8000ce9c64fd78cb6a33caed65e70 is 50, key is test_row_0/B:col10/1731620003630/Put/seqid=0 2024-11-14T21:33:25,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742123_1299 (size=12301) 2024-11-14T21:33:25,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/5ad8000ce9c64fd78cb6a33caed65e70 2024-11-14T21:33:25,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/a16ce2df481843f586baf29cca6af7ba is 50, key is test_row_0/C:col10/1731620003630/Put/seqid=0 2024-11-14T21:33:25,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742124_1300 (size=12301) 2024-11-14T21:33:25,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/a16ce2df481843f586baf29cca6af7ba 2024-11-14T21:33:25,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/a1c1fd42cfcc45a3b6d6b292b4b00186 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/a1c1fd42cfcc45a3b6d6b292b4b00186 2024-11-14T21:33:25,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/a1c1fd42cfcc45a3b6d6b292b4b00186, entries=150, sequenceid=408, filesize=12.0 K 2024-11-14T21:33:25,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/5ad8000ce9c64fd78cb6a33caed65e70 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/5ad8000ce9c64fd78cb6a33caed65e70 2024-11-14T21:33:25,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/5ad8000ce9c64fd78cb6a33caed65e70, entries=150, sequenceid=408, filesize=12.0 K 2024-11-14T21:33:25,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/a16ce2df481843f586baf29cca6af7ba as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/a16ce2df481843f586baf29cca6af7ba 2024-11-14T21:33:25,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/a16ce2df481843f586baf29cca6af7ba, entries=150, sequenceid=408, filesize=12.0 K 2024-11-14T21:33:25,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 99a2a424d1830a6f519f1971fbc9bf41 in 672ms, sequenceid=408, compaction requested=true 2024-11-14T21:33:25,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:25,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:33:25,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:25,450 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:25,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:33:25,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:25,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:33:25,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:33:25,450 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:25,463 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:25,464 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/A is initiating minor compaction (all files) 2024-11-14T21:33:25,464 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/A in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:25,464 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/dd8db7f9791446dd9ca9dd83c6da001b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/d5a9577e3f2649f98969abdcf098a49f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/a1c1fd42cfcc45a3b6d6b292b4b00186] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=36.8 K 2024-11-14T21:33:25,468 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting dd8db7f9791446dd9ca9dd83c6da001b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731620002057 2024-11-14T21:33:25,469 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:25,469 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/B is initiating minor compaction (all files) 2024-11-14T21:33:25,469 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/B in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:25,469 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/febded1128e54857ac0aaf372997d71b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/cd7dd8d49ff34e1c9e1d8ea3a58ac39f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/5ad8000ce9c64fd78cb6a33caed65e70] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=36.8 K 2024-11-14T21:33:25,469 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting d5a9577e3f2649f98969abdcf098a49f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731620002249 2024-11-14T21:33:25,471 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting a1c1fd42cfcc45a3b6d6b292b4b00186, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731620003590 2024-11-14T21:33:25,471 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting febded1128e54857ac0aaf372997d71b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731620002057 2024-11-14T21:33:25,474 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd7dd8d49ff34e1c9e1d8ea3a58ac39f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731620002249 2024-11-14T21:33:25,475 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ad8000ce9c64fd78cb6a33caed65e70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731620003590 2024-11-14T21:33:25,498 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#B#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:25,499 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/2167cd5d7ff849d2893ff602957dd757 is 50, key is test_row_0/B:col10/1731620003630/Put/seqid=0 2024-11-14T21:33:25,526 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#A#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:25,527 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/783f9ff0d8d94edeb3e46c6481c52dc1 is 50, key is test_row_0/A:col10/1731620003630/Put/seqid=0 2024-11-14T21:33:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:25,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T21:33:25,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:25,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:25,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:25,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:25,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:25,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:25,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742125_1301 (size=13221) 2024-11-14T21:33:25,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/b772ab937f0c490185a431d207a1adef is 50, key is test_row_0/A:col10/1731620004868/Put/seqid=0 2024-11-14T21:33:25,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742126_1302 (size=13221) 2024-11-14T21:33:25,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742127_1303 (size=12301) 2024-11-14T21:33:25,611 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/2167cd5d7ff849d2893ff602957dd757 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/2167cd5d7ff849d2893ff602957dd757 2024-11-14T21:33:25,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/b772ab937f0c490185a431d207a1adef 2024-11-14T21:33:25,615 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/783f9ff0d8d94edeb3e46c6481c52dc1 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/783f9ff0d8d94edeb3e46c6481c52dc1 2024-11-14T21:33:25,647 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/B of 99a2a424d1830a6f519f1971fbc9bf41 into 2167cd5d7ff849d2893ff602957dd757(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:25,647 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:25,647 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/B, priority=13, startTime=1731620005450; duration=0sec 2024-11-14T21:33:25,648 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:25,648 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:B 2024-11-14T21:33:25,648 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:25,659 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/A of 99a2a424d1830a6f519f1971fbc9bf41 into 783f9ff0d8d94edeb3e46c6481c52dc1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:25,659 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:25,659 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/A, priority=13, startTime=1731620005449; duration=0sec 2024-11-14T21:33:25,659 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:25,659 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:A 2024-11-14T21:33:25,672 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:25,672 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/C is initiating minor compaction (all files) 2024-11-14T21:33:25,672 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/C in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:25,672 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/d396a17479df44f7ba38ec47cf423341, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/96b5bbf371ce433f90df612e3a9966da, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/a16ce2df481843f586baf29cca6af7ba] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=36.8 K 2024-11-14T21:33:25,674 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting d396a17479df44f7ba38ec47cf423341, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731620002057 2024-11-14T21:33:25,675 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96b5bbf371ce433f90df612e3a9966da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731620002249 2024-11-14T21:33:25,678 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting a16ce2df481843f586baf29cca6af7ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731620003590 2024-11-14T21:33:25,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/be659a94510a4b3191baf0d362bdc2d6 is 50, key is test_row_0/B:col10/1731620004868/Put/seqid=0 2024-11-14T21:33:25,720 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#C#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:25,721 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/de893e2bd0d74aa79efb6e8e06f79e46 is 50, key is test_row_0/C:col10/1731620003630/Put/seqid=0 2024-11-14T21:33:25,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620065717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:25,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742128_1304 (size=12301) 2024-11-14T21:33:25,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/be659a94510a4b3191baf0d362bdc2d6 2024-11-14T21:33:25,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742129_1305 (size=13221) 2024-11-14T21:33:25,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/b24d684f22af4f9e9763e283042205d7 is 50, key is test_row_0/C:col10/1731620004868/Put/seqid=0 2024-11-14T21:33:25,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742130_1306 (size=12301) 2024-11-14T21:33:25,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:25,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620065830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:25,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:25,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:25,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:26,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620066046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:26,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:26,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:26,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:26,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/b24d684f22af4f9e9763e283042205d7 2024-11-14T21:33:26,221 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/de893e2bd0d74aa79efb6e8e06f79e46 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/de893e2bd0d74aa79efb6e8e06f79e46 2024-11-14T21:33:26,255 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/C of 99a2a424d1830a6f519f1971fbc9bf41 into de893e2bd0d74aa79efb6e8e06f79e46(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:26,255 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:26,255 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/C, priority=13, startTime=1731620005450; duration=0sec 2024-11-14T21:33:26,255 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:26,256 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:C 2024-11-14T21:33:26,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/b772ab937f0c490185a431d207a1adef as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/b772ab937f0c490185a431d207a1adef 2024-11-14T21:33:26,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/b772ab937f0c490185a431d207a1adef, entries=150, sequenceid=421, filesize=12.0 K 2024-11-14T21:33:26,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/be659a94510a4b3191baf0d362bdc2d6 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/be659a94510a4b3191baf0d362bdc2d6 2024-11-14T21:33:26,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/be659a94510a4b3191baf0d362bdc2d6, entries=150, sequenceid=421, filesize=12.0 K 2024-11-14T21:33:26,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/b24d684f22af4f9e9763e283042205d7 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/b24d684f22af4f9e9763e283042205d7 2024-11-14T21:33:26,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/b24d684f22af4f9e9763e283042205d7, entries=150, sequenceid=421, filesize=12.0 K 2024-11-14T21:33:26,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 99a2a424d1830a6f519f1971fbc9bf41 in 772ms, sequenceid=421, compaction requested=false 2024-11-14T21:33:26,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:26,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T21:33:26,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:26,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:26,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:26,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:26,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:26,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:26,375 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T21:33:26,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/fc1af359be294f6d843198feb0ef148e is 50, key is test_row_0/A:col10/1731620005706/Put/seqid=0 2024-11-14T21:33:26,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742131_1307 (size=14741) 2024-11-14T21:33:26,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/fc1af359be294f6d843198feb0ef148e 2024-11-14T21:33:26,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/a284a8f2e6b84dec9e30964c9f2e880e is 50, key is test_row_0/B:col10/1731620005706/Put/seqid=0 2024-11-14T21:33:26,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620066453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:26,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:26,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:26,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:26,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742132_1308 (size=12301) 2024-11-14T21:33:26,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/a284a8f2e6b84dec9e30964c9f2e880e 2024-11-14T21:33:26,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/97a34f5997254a6aa51ca1c1c1f57a7a is 50, key is test_row_0/C:col10/1731620005706/Put/seqid=0 2024-11-14T21:33:26,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742133_1309 (size=12301) 2024-11-14T21:33:26,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/97a34f5997254a6aa51ca1c1c1f57a7a 2024-11-14T21:33:26,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/fc1af359be294f6d843198feb0ef148e as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/fc1af359be294f6d843198feb0ef148e 2024-11-14T21:33:26,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/fc1af359be294f6d843198feb0ef148e, entries=200, sequenceid=444, filesize=14.4 K 2024-11-14T21:33:26,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/a284a8f2e6b84dec9e30964c9f2e880e as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a284a8f2e6b84dec9e30964c9f2e880e 2024-11-14T21:33:26,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a284a8f2e6b84dec9e30964c9f2e880e, entries=150, sequenceid=444, filesize=12.0 K 2024-11-14T21:33:26,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/97a34f5997254a6aa51ca1c1c1f57a7a as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/97a34f5997254a6aa51ca1c1c1f57a7a 2024-11-14T21:33:26,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/97a34f5997254a6aa51ca1c1c1f57a7a, entries=150, sequenceid=444, filesize=12.0 K 2024-11-14T21:33:26,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 99a2a424d1830a6f519f1971fbc9bf41 in 166ms, sequenceid=444, compaction requested=true 2024-11-14T21:33:26,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:26,536 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:26,537 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:26,537 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/A is initiating minor compaction (all files) 2024-11-14T21:33:26,538 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/A in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:26,538 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/783f9ff0d8d94edeb3e46c6481c52dc1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/b772ab937f0c490185a431d207a1adef, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/fc1af359be294f6d843198feb0ef148e] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=39.3 K 2024-11-14T21:33:26,538 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 783f9ff0d8d94edeb3e46c6481c52dc1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731620003590 2024-11-14T21:33:26,538 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting b772ab937f0c490185a431d207a1adef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731620004786 2024-11-14T21:33:26,539 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc1af359be294f6d843198feb0ef148e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731620005690 2024-11-14T21:33:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T21:33:26,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:33:26,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:26,541 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:26,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:33:26,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:26,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:33:26,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:26,542 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:26,543 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/B is initiating minor compaction (all files) 2024-11-14T21:33:26,543 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/B in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:26,543 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/2167cd5d7ff849d2893ff602957dd757, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/be659a94510a4b3191baf0d362bdc2d6, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a284a8f2e6b84dec9e30964c9f2e880e] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=36.9 K 2024-11-14T21:33:26,543 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 2167cd5d7ff849d2893ff602957dd757, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731620003590 2024-11-14T21:33:26,544 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting be659a94510a4b3191baf0d362bdc2d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731620004786 2024-11-14T21:33:26,544 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting a284a8f2e6b84dec9e30964c9f2e880e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731620005690 2024-11-14T21:33:26,547 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:33:26,547 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:33:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-14T21:33:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T21:33:26,551 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:33:26,552 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:33:26,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:33:26,561 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#A#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:26,562 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/b4d1a4ca47ab42bd9e6984ee6eb81957 is 50, key is test_row_0/A:col10/1731620005706/Put/seqid=0 2024-11-14T21:33:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:26,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T21:33:26,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:26,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:26,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:26,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:26,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:26,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:26,568 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#B#compaction#270 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:26,568 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/e23cadc43f3c4a8ab8799b11d67b40f1 is 50, key is test_row_0/B:col10/1731620005706/Put/seqid=0 2024-11-14T21:33:26,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/18335dbfaeb249299be002f4a77c7a4f is 50, key is test_row_0/A:col10/1731620006561/Put/seqid=0 2024-11-14T21:33:26,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742134_1310 (size=13323) 2024-11-14T21:33:26,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742135_1311 (size=13323) 2024-11-14T21:33:26,601 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/b4d1a4ca47ab42bd9e6984ee6eb81957 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/b4d1a4ca47ab42bd9e6984ee6eb81957 2024-11-14T21:33:26,602 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/e23cadc43f3c4a8ab8799b11d67b40f1 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/e23cadc43f3c4a8ab8799b11d67b40f1 2024-11-14T21:33:26,607 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/A of 99a2a424d1830a6f519f1971fbc9bf41 into b4d1a4ca47ab42bd9e6984ee6eb81957(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:26,607 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:26,607 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/A, priority=13, startTime=1731620006536; duration=0sec 2024-11-14T21:33:26,608 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:26,608 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:A 2024-11-14T21:33:26,608 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:26,609 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/B of 99a2a424d1830a6f519f1971fbc9bf41 into e23cadc43f3c4a8ab8799b11d67b40f1(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:26,609 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:26,609 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/B, priority=13, startTime=1731620006541; duration=0sec 2024-11-14T21:33:26,609 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:26,609 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:B 2024-11-14T21:33:26,609 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:26,610 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/C is initiating minor compaction (all files) 2024-11-14T21:33:26,610 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/C in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:26,610 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/de893e2bd0d74aa79efb6e8e06f79e46, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/b24d684f22af4f9e9763e283042205d7, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/97a34f5997254a6aa51ca1c1c1f57a7a] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=36.9 K 2024-11-14T21:33:26,610 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting de893e2bd0d74aa79efb6e8e06f79e46, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731620003590 2024-11-14T21:33:26,610 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting b24d684f22af4f9e9763e283042205d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731620004786 2024-11-14T21:33:26,611 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97a34f5997254a6aa51ca1c1c1f57a7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731620005690 2024-11-14T21:33:26,621 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#C#compaction#272 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:26,621 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/b7ff3588b92d481c91a12be69a6941bf is 50, key is test_row_0/C:col10/1731620005706/Put/seqid=0 2024-11-14T21:33:26,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742136_1312 (size=12301) 2024-11-14T21:33:26,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742137_1313 (size=13323) 2024-11-14T21:33:26,636 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/b7ff3588b92d481c91a12be69a6941bf as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/b7ff3588b92d481c91a12be69a6941bf 2024-11-14T21:33:26,643 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/C of 99a2a424d1830a6f519f1971fbc9bf41 into b7ff3588b92d481c91a12be69a6941bf(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:26,643 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:26,643 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/C, priority=13, startTime=1731620006542; duration=0sec 2024-11-14T21:33:26,643 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:26,643 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:C 2024-11-14T21:33:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T21:33:26,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620066663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:26,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:26,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:26,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:26,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T21:33:26,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:26,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:26,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:26,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:26,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:26,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620066775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:26,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:26,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:26,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T21:33:26,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T21:33:26,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:26,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:26,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:26,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:26,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:26,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620066982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:26,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:26,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:26,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:27,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46332 deadline: 1731620067000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:27,005 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_0' in 99a2a424d1830a6f519f1971fbc9bf41 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:33:27,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:27,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:27,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:27,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:27,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/18335dbfaeb249299be002f4a77c7a4f 2024-11-14T21:33:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46346 deadline: 1731620067023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:27,024 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_2' in 99a2a424d1830a6f519f1971fbc9bf41 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:33:27,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:27,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:27,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:27,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T21:33:27,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/eab78ff6c5924cec8f2c413fe4badcb4 is 50, key is test_row_0/B:col10/1731620006561/Put/seqid=0 2024-11-14T21:33:27,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:27,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:27,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:27,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:27,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:27,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742138_1314 (size=12301) 2024-11-14T21:33:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T21:33:27,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T21:33:27,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:27,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:27,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:27,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:27,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:27,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620067291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:27,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:27,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:27,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:27,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T21:33:27,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:27,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:27,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:27,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:27,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:27,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/eab78ff6c5924cec8f2c413fe4badcb4 2024-11-14T21:33:27,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/cbed77af6702476da272dd7769f19e13 is 50, key is test_row_0/C:col10/1731620006561/Put/seqid=0 2024-11-14T21:33:27,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T21:33:27,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:27,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:27,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:27,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:27,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:27,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742139_1315 (size=12301) 2024-11-14T21:33:27,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/cbed77af6702476da272dd7769f19e13 2024-11-14T21:33:27,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/18335dbfaeb249299be002f4a77c7a4f as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/18335dbfaeb249299be002f4a77c7a4f 2024-11-14T21:33:27,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/18335dbfaeb249299be002f4a77c7a4f, entries=150, sequenceid=457, filesize=12.0 K 2024-11-14T21:33:27,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/eab78ff6c5924cec8f2c413fe4badcb4 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/eab78ff6c5924cec8f2c413fe4badcb4 2024-11-14T21:33:27,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/eab78ff6c5924cec8f2c413fe4badcb4, entries=150, sequenceid=457, filesize=12.0 K 2024-11-14T21:33:27,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/cbed77af6702476da272dd7769f19e13 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/cbed77af6702476da272dd7769f19e13 2024-11-14T21:33:27,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/cbed77af6702476da272dd7769f19e13, entries=150, sequenceid=457, filesize=12.0 K 2024-11-14T21:33:27,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 99a2a424d1830a6f519f1971fbc9bf41 in 1027ms, sequenceid=457, compaction requested=false 2024-11-14T21:33:27,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:27,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T21:33:27,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:27,687 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T21:33:27,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:27,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:27,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:27,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:27,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:27,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T21:33:27,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/4d0bc02720974948a354942dd4d77886 is 50, key is test_row_0/A:col10/1731620006649/Put/seqid=0 2024-11-14T21:33:27,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742140_1316 (size=12301) 2024-11-14T21:33:27,717 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/4d0bc02720974948a354942dd4d77886 2024-11-14T21:33:27,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/61d0a66564e74ccfad78bce40bc179bc is 50, key is test_row_0/B:col10/1731620006649/Put/seqid=0 2024-11-14T21:33:27,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742141_1317 (size=12301) 2024-11-14T21:33:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:27,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:27,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620067864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:27,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:27,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:27,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:27,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620067981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:28,158 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/61d0a66564e74ccfad78bce40bc179bc 2024-11-14T21:33:28,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620068190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:28,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:28,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:28,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:28,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/0da65d73bdbe4d498e18427590c69ecd is 50, key is test_row_0/C:col10/1731620006649/Put/seqid=0 2024-11-14T21:33:28,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742142_1318 (size=12301) 2024-11-14T21:33:28,247 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/0da65d73bdbe4d498e18427590c69ecd 2024-11-14T21:33:28,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/4d0bc02720974948a354942dd4d77886 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/4d0bc02720974948a354942dd4d77886 2024-11-14T21:33:28,291 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/4d0bc02720974948a354942dd4d77886, entries=150, sequenceid=479, filesize=12.0 K 2024-11-14T21:33:28,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/61d0a66564e74ccfad78bce40bc179bc as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/61d0a66564e74ccfad78bce40bc179bc 2024-11-14T21:33:28,303 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/61d0a66564e74ccfad78bce40bc179bc, entries=150, sequenceid=479, filesize=12.0 K 2024-11-14T21:33:28,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/0da65d73bdbe4d498e18427590c69ecd as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/0da65d73bdbe4d498e18427590c69ecd 2024-11-14T21:33:28,309 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/0da65d73bdbe4d498e18427590c69ecd, entries=150, sequenceid=479, filesize=12.0 K 2024-11-14T21:33:28,314 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 99a2a424d1830a6f519f1971fbc9bf41 in 627ms, sequenceid=479, compaction requested=true 2024-11-14T21:33:28,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:28,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:28,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-14T21:33:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-14T21:33:28,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-14T21:33:28,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7730 sec 2024-11-14T21:33:28,340 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.7830 sec 2024-11-14T21:33:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:28,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T21:33:28,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:28,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:28,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:28,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:28,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:28,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:28,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/94c643269e464ba1a3cc146c2ab59d86 is 50, key is test_row_0/A:col10/1731620008505/Put/seqid=0 2024-11-14T21:33:28,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742143_1319 (size=12301) 2024-11-14T21:33:28,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/94c643269e464ba1a3cc146c2ab59d86 2024-11-14T21:33:28,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/db652e96748d434f811431ba3eb4f721 is 50, key is test_row_0/B:col10/1731620008505/Put/seqid=0 2024-11-14T21:33:28,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620068654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:28,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742144_1320 (size=12301) 2024-11-14T21:33:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T21:33:28,707 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:33:28,708 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:33:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-14T21:33:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T21:33:28,730 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:33:28,732 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:33:28,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:33:28,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620068771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T21:33:28,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T21:33:28,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:28,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:28,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:28,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:28,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:28,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:28,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620068980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:29,029 INFO [Thread-1048 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1048. 2024-11-14T21:33:29,029 DEBUG [Thread-1048 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:33:29,029 DEBUG [Thread-1048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:29,029 DEBUG [Thread-1048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:29,029 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:33:29,031 INFO [Thread-1050 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1050. 2024-11-14T21:33:29,031 DEBUG [Thread-1050 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:33:29,031 DEBUG [Thread-1050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:29,031 DEBUG [Thread-1050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:29,031 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:33:29,033 INFO [Thread-1056 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1056. 2024-11-14T21:33:29,033 DEBUG [Thread-1056 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:33:29,033 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:29,033 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:29,033 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:33:29,034 INFO [Thread-1054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1054. 2024-11-14T21:33:29,034 DEBUG [Thread-1054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:33:29,034 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:29,034 INFO [Thread-1052 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1052. 2024-11-14T21:33:29,035 DEBUG [Thread-1052 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:33:29,035 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:29,035 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:29,035 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:33:29,035 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:29,035 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:33:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T21:33:29,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T21:33:29,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:29,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:29,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:29,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:29,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:29,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/db652e96748d434f811431ba3eb4f721 2024-11-14T21:33:29,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/36e2088cfb3d4c6b98b6f918409470e4 is 50, key is test_row_0/C:col10/1731620008505/Put/seqid=0 2024-11-14T21:33:29,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742145_1321 (size=12301) 2024-11-14T21:33:29,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T21:33:29,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:29,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:29,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:29,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:29,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:29,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46362 deadline: 1731620069290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:29,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:29,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:29,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:33:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T21:33:29,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T21:33:29,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:29,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:29,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:29,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:29,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:29,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/36e2088cfb3d4c6b98b6f918409470e4 2024-11-14T21:33:29,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/94c643269e464ba1a3cc146c2ab59d86 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/94c643269e464ba1a3cc146c2ab59d86 2024-11-14T21:33:29,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/94c643269e464ba1a3cc146c2ab59d86, entries=150, sequenceid=493, filesize=12.0 K 2024-11-14T21:33:29,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/db652e96748d434f811431ba3eb4f721 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/db652e96748d434f811431ba3eb4f721 2024-11-14T21:33:29,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/db652e96748d434f811431ba3eb4f721, entries=150, sequenceid=493, filesize=12.0 K 2024-11-14T21:33:29,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/36e2088cfb3d4c6b98b6f918409470e4 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/36e2088cfb3d4c6b98b6f918409470e4 2024-11-14T21:33:29,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/36e2088cfb3d4c6b98b6f918409470e4, entries=150, sequenceid=493, filesize=12.0 K 2024-11-14T21:33:29,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 99a2a424d1830a6f519f1971fbc9bf41 in 993ms, sequenceid=493, compaction requested=true 2024-11-14T21:33:29,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:29,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:33:29,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:29,512 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:33:29,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:33:29,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:29,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99a2a424d1830a6f519f1971fbc9bf41:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:33:29,512 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:33:29,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:29,513 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:33:29,513 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:33:29,513 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/A is initiating minor compaction (all files) 2024-11-14T21:33:29,513 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/B is initiating minor compaction (all files) 2024-11-14T21:33:29,513 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/A in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:29,513 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/B in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:29,513 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/b4d1a4ca47ab42bd9e6984ee6eb81957, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/18335dbfaeb249299be002f4a77c7a4f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/4d0bc02720974948a354942dd4d77886, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/94c643269e464ba1a3cc146c2ab59d86] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=49.0 K 2024-11-14T21:33:29,513 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/e23cadc43f3c4a8ab8799b11d67b40f1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/eab78ff6c5924cec8f2c413fe4badcb4, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/61d0a66564e74ccfad78bce40bc179bc, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/db652e96748d434f811431ba3eb4f721] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=49.0 K 2024-11-14T21:33:29,514 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4d1a4ca47ab42bd9e6984ee6eb81957, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731620005690 2024-11-14T21:33:29,514 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting e23cadc43f3c4a8ab8799b11d67b40f1, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731620005690 2024-11-14T21:33:29,514 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting eab78ff6c5924cec8f2c413fe4badcb4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731620006416 2024-11-14T21:33:29,514 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18335dbfaeb249299be002f4a77c7a4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731620006416 2024-11-14T21:33:29,515 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d0bc02720974948a354942dd4d77886, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731620006649 2024-11-14T21:33:29,515 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 61d0a66564e74ccfad78bce40bc179bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731620006649 2024-11-14T21:33:29,515 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94c643269e464ba1a3cc146c2ab59d86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731620007852 2024-11-14T21:33:29,515 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting db652e96748d434f811431ba3eb4f721, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731620007852 2024-11-14T21:33:29,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T21:33:29,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:29,524 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T21:33:29,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:29,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:29,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:29,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:29,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:29,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:29,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/e3e033e8af9f4c76ac0c0ca0dd1c1f44 is 50, key is test_row_0/A:col10/1731620008651/Put/seqid=0 2024-11-14T21:33:29,545 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#A#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:29,545 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/0fd15ea946cf44eba945924ce2e14df3 is 50, key is test_row_0/A:col10/1731620008505/Put/seqid=0 2024-11-14T21:33:29,545 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#B#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:29,546 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/28a123df488a4fd89e64924162cf2bcd is 50, key is test_row_0/B:col10/1731620008505/Put/seqid=0 2024-11-14T21:33:29,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742146_1322 (size=12301) 2024-11-14T21:33:29,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742148_1324 (size=13459) 2024-11-14T21:33:29,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742147_1323 (size=13459) 2024-11-14T21:33:29,571 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/28a123df488a4fd89e64924162cf2bcd as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/28a123df488a4fd89e64924162cf2bcd 2024-11-14T21:33:29,584 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/B of 99a2a424d1830a6f519f1971fbc9bf41 into 28a123df488a4fd89e64924162cf2bcd(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:29,584 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:29,584 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/B, priority=12, startTime=1731620009512; duration=0sec 2024-11-14T21:33:29,584 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:29,584 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:B 2024-11-14T21:33:29,584 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:33:29,587 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:33:29,587 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 99a2a424d1830a6f519f1971fbc9bf41/C is initiating minor compaction (all files) 2024-11-14T21:33:29,588 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99a2a424d1830a6f519f1971fbc9bf41/C in TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:29,588 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/b7ff3588b92d481c91a12be69a6941bf, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/cbed77af6702476da272dd7769f19e13, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/0da65d73bdbe4d498e18427590c69ecd, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/36e2088cfb3d4c6b98b6f918409470e4] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp, totalSize=49.0 K 2024-11-14T21:33:29,588 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting b7ff3588b92d481c91a12be69a6941bf, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731620005690 2024-11-14T21:33:29,588 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting cbed77af6702476da272dd7769f19e13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731620006416 2024-11-14T21:33:29,589 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 0da65d73bdbe4d498e18427590c69ecd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731620006649 2024-11-14T21:33:29,592 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 36e2088cfb3d4c6b98b6f918409470e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731620007852 2024-11-14T21:33:29,616 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99a2a424d1830a6f519f1971fbc9bf41#C#compaction#284 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:29,616 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/927fe9bb565c4a7d9c1bb80ab9ff366f is 50, key is test_row_0/C:col10/1731620008505/Put/seqid=0 2024-11-14T21:33:29,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742149_1325 (size=13459) 2024-11-14T21:33:29,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:29,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. as already flushing 2024-11-14T21:33:29,802 INFO [Thread-1043 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1043. 2024-11-14T21:33:29,802 DEBUG [Thread-1043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:33:29,802 DEBUG [Thread-1043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:29,802 DEBUG [Thread-1043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:29,802 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:33:29,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T21:33:29,946 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/e3e033e8af9f4c76ac0c0ca0dd1c1f44 2024-11-14T21:33:29,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/1a60b9b1ff5543a48470107178e5041c is 50, key is test_row_0/B:col10/1731620008651/Put/seqid=0 2024-11-14T21:33:29,977 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/0fd15ea946cf44eba945924ce2e14df3 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/0fd15ea946cf44eba945924ce2e14df3 2024-11-14T21:33:29,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742150_1326 (size=12301) 2024-11-14T21:33:29,995 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/A of 99a2a424d1830a6f519f1971fbc9bf41 into 0fd15ea946cf44eba945924ce2e14df3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:29,995 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:29,995 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/A, priority=12, startTime=1731620009512; duration=0sec 2024-11-14T21:33:29,995 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:29,995 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:A 2024-11-14T21:33:30,089 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/927fe9bb565c4a7d9c1bb80ab9ff366f as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/927fe9bb565c4a7d9c1bb80ab9ff366f 2024-11-14T21:33:30,094 INFO [Thread-1039 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1039. 2024-11-14T21:33:30,094 DEBUG [Thread-1039 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:33:30,094 DEBUG [Thread-1039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:30,095 DEBUG [Thread-1039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:30,095 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:33:30,123 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 99a2a424d1830a6f519f1971fbc9bf41/C of 99a2a424d1830a6f519f1971fbc9bf41 into 927fe9bb565c4a7d9c1bb80ab9ff366f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:30,123 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:30,123 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., storeName=99a2a424d1830a6f519f1971fbc9bf41/C, priority=12, startTime=1731620009512; duration=0sec 2024-11-14T21:33:30,123 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:30,123 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99a2a424d1830a6f519f1971fbc9bf41:C 2024-11-14T21:33:30,153 INFO [Thread-1045 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1045. 2024-11-14T21:33:30,153 DEBUG [Thread-1045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:33:30,153 DEBUG [Thread-1045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:30,153 DEBUG [Thread-1045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:30,153 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:33:30,387 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/1a60b9b1ff5543a48470107178e5041c 2024-11-14T21:33:30,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/3a7ec5b51e5849048557ca243433f545 is 50, key is test_row_0/C:col10/1731620008651/Put/seqid=0 2024-11-14T21:33:30,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742151_1327 (size=12301) 2024-11-14T21:33:30,799 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/3a7ec5b51e5849048557ca243433f545 2024-11-14T21:33:30,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/e3e033e8af9f4c76ac0c0ca0dd1c1f44 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/e3e033e8af9f4c76ac0c0ca0dd1c1f44 2024-11-14T21:33:30,807 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/e3e033e8af9f4c76ac0c0ca0dd1c1f44, entries=150, sequenceid=511, filesize=12.0 K 2024-11-14T21:33:30,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/1a60b9b1ff5543a48470107178e5041c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/1a60b9b1ff5543a48470107178e5041c 2024-11-14T21:33:30,812 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/1a60b9b1ff5543a48470107178e5041c, entries=150, sequenceid=511, filesize=12.0 K 2024-11-14T21:33:30,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/3a7ec5b51e5849048557ca243433f545 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/3a7ec5b51e5849048557ca243433f545 2024-11-14T21:33:30,816 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/3a7ec5b51e5849048557ca243433f545, entries=150, sequenceid=511, filesize=12.0 K 2024-11-14T21:33:30,817 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=20.13 KB/20610 for 99a2a424d1830a6f519f1971fbc9bf41 in 1293ms, sequenceid=511, compaction requested=false 2024-11-14T21:33:30,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 99a2a424d1830a6f519f1971fbc9bf41: 2024-11-14T21:33:30,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:30,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-14T21:33:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-14T21:33:30,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-14T21:33:30,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0870 sec 2024-11-14T21:33:30,823 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.1130 sec 2024-11-14T21:33:30,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T21:33:30,871 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:33:37,083 INFO [Thread-1037 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1037. 2024-11-14T21:33:37,083 DEBUG [Thread-1037 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:33:37,083 DEBUG [Thread-1037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:37,083 DEBUG [Thread-1037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:37,084 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:33:37,102 INFO [Thread-1041 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1041. 2024-11-14T21:33:37,102 DEBUG [Thread-1041 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:33:37,102 DEBUG [Thread-1041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:37,103 DEBUG [Thread-1041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:37,103 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:33:37,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T21:33:37,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-14T21:33:37,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-14T21:33:37,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-14T21:33:37,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 206 2024-11-14T21:33:37,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-14T21:33:37,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T21:33:37,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1954 2024-11-14T21:33:37,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1959 2024-11-14T21:33:37,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1874 2024-11-14T21:33:37,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1928 2024-11-14T21:33:37,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1954 2024-11-14T21:33:37,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T21:33:37,103 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T21:33:37,103 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T21:33:37,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:37,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:37,104 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:33:37,105 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T21:33:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T21:33:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T21:33:37,108 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731620017108"}]},"ts":"1731620017108"} 2024-11-14T21:33:37,111 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T21:33:37,111 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T21:33:37,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T21:33:37,113 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=99a2a424d1830a6f519f1971fbc9bf41, UNASSIGN}] 2024-11-14T21:33:37,114 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=99a2a424d1830a6f519f1971fbc9bf41, UNASSIGN 2024-11-14T21:33:37,115 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=99a2a424d1830a6f519f1971fbc9bf41, regionState=CLOSING, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:37,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=99a2a424d1830a6f519f1971fbc9bf41, UNASSIGN because future has completed 2024-11-14T21:33:37,117 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T21:33:37,117 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; CloseRegionProcedure 99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560}] 2024-11-14T21:33:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T21:33:37,270 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] handler.UnassignRegionHandler(122): Close 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:37,270 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T21:33:37,270 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1722): Closing 99a2a424d1830a6f519f1971fbc9bf41, disabling compactions & flushes 2024-11-14T21:33:37,270 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:37,270 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:37,270 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. after waiting 0 ms 2024-11-14T21:33:37,270 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:37,270 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(2902): Flushing 99a2a424d1830a6f519f1971fbc9bf41 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T21:33:37,271 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=A 2024-11-14T21:33:37,271 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:37,271 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=B 2024-11-14T21:33:37,271 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:37,271 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 99a2a424d1830a6f519f1971fbc9bf41, store=C 2024-11-14T21:33:37,271 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:37,276 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/63182b6de0314a398446549a1768cb63 is 50, key is test_row_0/A:col10/1731620017100/Put/seqid=0 2024-11-14T21:33:37,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742152_1328 (size=12301) 2024-11-14T21:33:37,282 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/63182b6de0314a398446549a1768cb63 2024-11-14T21:33:37,291 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/5321193cfb764764ae8c9128334d05c5 is 50, key is test_row_0/B:col10/1731620017100/Put/seqid=0 2024-11-14T21:33:37,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742153_1329 (size=12301) 2024-11-14T21:33:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T21:33:37,695 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/5321193cfb764764ae8c9128334d05c5 2024-11-14T21:33:37,702 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/2fd02b5252ad400c988bb044a2da10a9 is 50, key is test_row_0/C:col10/1731620017100/Put/seqid=0 2024-11-14T21:33:37,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742154_1330 (size=12301) 2024-11-14T21:33:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T21:33:38,118 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/2fd02b5252ad400c988bb044a2da10a9 2024-11-14T21:33:38,147 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/A/63182b6de0314a398446549a1768cb63 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/63182b6de0314a398446549a1768cb63 2024-11-14T21:33:38,152 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/63182b6de0314a398446549a1768cb63, entries=150, sequenceid=522, filesize=12.0 K 2024-11-14T21:33:38,154 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/B/5321193cfb764764ae8c9128334d05c5 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/5321193cfb764764ae8c9128334d05c5 2024-11-14T21:33:38,159 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/5321193cfb764764ae8c9128334d05c5, entries=150, sequenceid=522, filesize=12.0 K 2024-11-14T21:33:38,160 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/.tmp/C/2fd02b5252ad400c988bb044a2da10a9 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/2fd02b5252ad400c988bb044a2da10a9 2024-11-14T21:33:38,167 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/2fd02b5252ad400c988bb044a2da10a9, entries=150, sequenceid=522, filesize=12.0 K 2024-11-14T21:33:38,174 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 99a2a424d1830a6f519f1971fbc9bf41 in 903ms, sequenceid=522, compaction requested=true 2024-11-14T21:33:38,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/ef83ecfe35104a9a98f1b9ce1cc76afe, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/4f33fa600c01405aa40dc89ac6b1bf93, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/7fe67b9e8fb14da2b9175c7d851397ff, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/52ba67a38c9549a4838d78aad00080f3, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/617291120c74496da68943fdce46ddfe, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/42cf552ebab946928b08a9590d50a6c6, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/aff7771bdf714466b18483c08dee4bce, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/2d29c1da398e4e1d803c85a736e4c34e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/03c3453974654ea2b7041ad775be87ce, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/b1a99692dd404dee8fcc23d49696e18c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/ee972a4c9d9145ffa14e68156524646d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/4189428e0b8640bd811c9400e1d45ecc, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/6e35fe2a084a40cb8514d5e4a5db2839, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/f2ec75d36fc94014826a54710ef7635b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/bf9eeee2f25e4b5bbc95d7a2e0f52169, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/ca96259b7942495ab95620272138fd05, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/8b56b496b9f34a45a4d65f7739874a0d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/e455527c6763454293b4b3051c9d548c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/c51f23a7dc444a7d8f1587d2db1c55ab, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/74848ce380c141e88cdc35c02fcdd197, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/155c50021693490fa8435e597a839640, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/f5644b35cd624c039abc894aba0ec960, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/90b0d5506b6a4a88b76c6f0935765dd4, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/dd8db7f9791446dd9ca9dd83c6da001b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/0645f15a4a184ba29aed7edf65786005, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/d5a9577e3f2649f98969abdcf098a49f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/783f9ff0d8d94edeb3e46c6481c52dc1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/a1c1fd42cfcc45a3b6d6b292b4b00186, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/b772ab937f0c490185a431d207a1adef, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/fc1af359be294f6d843198feb0ef148e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/b4d1a4ca47ab42bd9e6984ee6eb81957, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/18335dbfaeb249299be002f4a77c7a4f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/4d0bc02720974948a354942dd4d77886, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/94c643269e464ba1a3cc146c2ab59d86] to archive 2024-11-14T21:33:38,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:33:38,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/ef83ecfe35104a9a98f1b9ce1cc76afe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/ef83ecfe35104a9a98f1b9ce1cc76afe 2024-11-14T21:33:38,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/4f33fa600c01405aa40dc89ac6b1bf93 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/4f33fa600c01405aa40dc89ac6b1bf93 2024-11-14T21:33:38,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/7fe67b9e8fb14da2b9175c7d851397ff to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/7fe67b9e8fb14da2b9175c7d851397ff 2024-11-14T21:33:38,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/52ba67a38c9549a4838d78aad00080f3 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/52ba67a38c9549a4838d78aad00080f3 2024-11-14T21:33:38,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/617291120c74496da68943fdce46ddfe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/617291120c74496da68943fdce46ddfe 2024-11-14T21:33:38,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/42cf552ebab946928b08a9590d50a6c6 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/42cf552ebab946928b08a9590d50a6c6 2024-11-14T21:33:38,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/aff7771bdf714466b18483c08dee4bce to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/aff7771bdf714466b18483c08dee4bce 2024-11-14T21:33:38,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/2d29c1da398e4e1d803c85a736e4c34e to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/2d29c1da398e4e1d803c85a736e4c34e 2024-11-14T21:33:38,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T21:33:38,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/03c3453974654ea2b7041ad775be87ce to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/03c3453974654ea2b7041ad775be87ce 2024-11-14T21:33:38,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/b1a99692dd404dee8fcc23d49696e18c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/b1a99692dd404dee8fcc23d49696e18c 2024-11-14T21:33:38,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/ee972a4c9d9145ffa14e68156524646d to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/ee972a4c9d9145ffa14e68156524646d 2024-11-14T21:33:38,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/4189428e0b8640bd811c9400e1d45ecc to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/4189428e0b8640bd811c9400e1d45ecc 2024-11-14T21:33:38,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/6e35fe2a084a40cb8514d5e4a5db2839 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/6e35fe2a084a40cb8514d5e4a5db2839 2024-11-14T21:33:38,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/f2ec75d36fc94014826a54710ef7635b to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/f2ec75d36fc94014826a54710ef7635b 2024-11-14T21:33:38,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/bf9eeee2f25e4b5bbc95d7a2e0f52169 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/bf9eeee2f25e4b5bbc95d7a2e0f52169 2024-11-14T21:33:38,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/ca96259b7942495ab95620272138fd05 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/ca96259b7942495ab95620272138fd05 2024-11-14T21:33:38,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/8b56b496b9f34a45a4d65f7739874a0d to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/8b56b496b9f34a45a4d65f7739874a0d 2024-11-14T21:33:38,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/e455527c6763454293b4b3051c9d548c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/e455527c6763454293b4b3051c9d548c 2024-11-14T21:33:38,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/c51f23a7dc444a7d8f1587d2db1c55ab to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/c51f23a7dc444a7d8f1587d2db1c55ab 2024-11-14T21:33:38,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/74848ce380c141e88cdc35c02fcdd197 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/74848ce380c141e88cdc35c02fcdd197 2024-11-14T21:33:38,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/155c50021693490fa8435e597a839640 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/155c50021693490fa8435e597a839640 2024-11-14T21:33:38,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/f5644b35cd624c039abc894aba0ec960 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/f5644b35cd624c039abc894aba0ec960 2024-11-14T21:33:38,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/90b0d5506b6a4a88b76c6f0935765dd4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/90b0d5506b6a4a88b76c6f0935765dd4 2024-11-14T21:33:38,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/dd8db7f9791446dd9ca9dd83c6da001b to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/dd8db7f9791446dd9ca9dd83c6da001b 2024-11-14T21:33:38,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/0645f15a4a184ba29aed7edf65786005 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/0645f15a4a184ba29aed7edf65786005 2024-11-14T21:33:38,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/d5a9577e3f2649f98969abdcf098a49f to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/d5a9577e3f2649f98969abdcf098a49f 2024-11-14T21:33:38,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/783f9ff0d8d94edeb3e46c6481c52dc1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/783f9ff0d8d94edeb3e46c6481c52dc1 2024-11-14T21:33:38,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/a1c1fd42cfcc45a3b6d6b292b4b00186 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/a1c1fd42cfcc45a3b6d6b292b4b00186 2024-11-14T21:33:38,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/b772ab937f0c490185a431d207a1adef to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/b772ab937f0c490185a431d207a1adef 2024-11-14T21:33:38,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/fc1af359be294f6d843198feb0ef148e to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/fc1af359be294f6d843198feb0ef148e 2024-11-14T21:33:38,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/b4d1a4ca47ab42bd9e6984ee6eb81957 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/b4d1a4ca47ab42bd9e6984ee6eb81957 2024-11-14T21:33:38,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/18335dbfaeb249299be002f4a77c7a4f to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/18335dbfaeb249299be002f4a77c7a4f 2024-11-14T21:33:38,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/4d0bc02720974948a354942dd4d77886 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/4d0bc02720974948a354942dd4d77886 2024-11-14T21:33:38,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/94c643269e464ba1a3cc146c2ab59d86 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/94c643269e464ba1a3cc146c2ab59d86 2024-11-14T21:33:38,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a9a04497da4842c8a3505e80fc4d68b7, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/c76c463bdcf2437f94dae27af3fb0414, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a94e00f7f574456a8734465a265692fe, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/8842380cce6b4e33b19e0cda90a9c9a3, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a446d83fdced4ceca7852886e5ab9ab5, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/144af65ca3d14775b9889353bc47cb10, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/714b8b9639574954be8fe91a48a1d248, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/f5f0fde12be14dbba0b0ecff515fbcd8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/be16da7816b24591a73a1c339ba8f0c9, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b0525bc3c0a04c9dbbf3f97ed624df46, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/77343de8969a4e4c968c155ea1db1a17, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/50e3b94af4e64aa6aa6664d621b91cd9, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/8e0302d68f4e4f2c820cab3e3beb03e4, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/d85dd532cecd4992b0b76ec759bb933b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/931647dd849144478dce9cbee885f627, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/11ed7269b6a443d8a64a79af830a97b7, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/941298d6df984aec8602b86d9d729875, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/efa5253aca3b4f55b89845017ed903ed, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/9e160f4a03164f8986e575bc12c0d6a1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/7524c6d199584bd8b78b776630019d10, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/abcbfc8d90824778a6105289487fb174, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b27d7f44dfab4983a52b6bbb61abb83c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b743cbc2fbe54072bbec5734f9785061, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/febded1128e54857ac0aaf372997d71b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b2e095b5775442cf899f267528918010, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/cd7dd8d49ff34e1c9e1d8ea3a58ac39f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/2167cd5d7ff849d2893ff602957dd757, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/5ad8000ce9c64fd78cb6a33caed65e70, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/be659a94510a4b3191baf0d362bdc2d6, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/e23cadc43f3c4a8ab8799b11d67b40f1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a284a8f2e6b84dec9e30964c9f2e880e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/eab78ff6c5924cec8f2c413fe4badcb4, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/61d0a66564e74ccfad78bce40bc179bc, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/db652e96748d434f811431ba3eb4f721] to archive 2024-11-14T21:33:38,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:33:38,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a9a04497da4842c8a3505e80fc4d68b7 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a9a04497da4842c8a3505e80fc4d68b7 2024-11-14T21:33:38,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/c76c463bdcf2437f94dae27af3fb0414 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/c76c463bdcf2437f94dae27af3fb0414 2024-11-14T21:33:38,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a94e00f7f574456a8734465a265692fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a94e00f7f574456a8734465a265692fe 2024-11-14T21:33:38,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/8842380cce6b4e33b19e0cda90a9c9a3 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/8842380cce6b4e33b19e0cda90a9c9a3 2024-11-14T21:33:38,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a446d83fdced4ceca7852886e5ab9ab5 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a446d83fdced4ceca7852886e5ab9ab5 2024-11-14T21:33:38,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/144af65ca3d14775b9889353bc47cb10 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/144af65ca3d14775b9889353bc47cb10 2024-11-14T21:33:38,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/714b8b9639574954be8fe91a48a1d248 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/714b8b9639574954be8fe91a48a1d248 2024-11-14T21:33:38,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/f5f0fde12be14dbba0b0ecff515fbcd8 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/f5f0fde12be14dbba0b0ecff515fbcd8 2024-11-14T21:33:38,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/be16da7816b24591a73a1c339ba8f0c9 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/be16da7816b24591a73a1c339ba8f0c9 2024-11-14T21:33:38,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b0525bc3c0a04c9dbbf3f97ed624df46 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b0525bc3c0a04c9dbbf3f97ed624df46 2024-11-14T21:33:38,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/77343de8969a4e4c968c155ea1db1a17 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/77343de8969a4e4c968c155ea1db1a17 2024-11-14T21:33:38,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/50e3b94af4e64aa6aa6664d621b91cd9 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/50e3b94af4e64aa6aa6664d621b91cd9 2024-11-14T21:33:38,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/8e0302d68f4e4f2c820cab3e3beb03e4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/8e0302d68f4e4f2c820cab3e3beb03e4 2024-11-14T21:33:38,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/d85dd532cecd4992b0b76ec759bb933b to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/d85dd532cecd4992b0b76ec759bb933b 2024-11-14T21:33:38,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/931647dd849144478dce9cbee885f627 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/931647dd849144478dce9cbee885f627 2024-11-14T21:33:38,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/11ed7269b6a443d8a64a79af830a97b7 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/11ed7269b6a443d8a64a79af830a97b7 2024-11-14T21:33:38,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/941298d6df984aec8602b86d9d729875 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/941298d6df984aec8602b86d9d729875 2024-11-14T21:33:38,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/efa5253aca3b4f55b89845017ed903ed to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/efa5253aca3b4f55b89845017ed903ed 2024-11-14T21:33:38,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/9e160f4a03164f8986e575bc12c0d6a1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/9e160f4a03164f8986e575bc12c0d6a1 2024-11-14T21:33:38,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/7524c6d199584bd8b78b776630019d10 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/7524c6d199584bd8b78b776630019d10 2024-11-14T21:33:38,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/abcbfc8d90824778a6105289487fb174 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/abcbfc8d90824778a6105289487fb174 2024-11-14T21:33:38,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b27d7f44dfab4983a52b6bbb61abb83c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b27d7f44dfab4983a52b6bbb61abb83c 2024-11-14T21:33:38,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b743cbc2fbe54072bbec5734f9785061 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b743cbc2fbe54072bbec5734f9785061 2024-11-14T21:33:38,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/febded1128e54857ac0aaf372997d71b to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/febded1128e54857ac0aaf372997d71b 2024-11-14T21:33:38,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b2e095b5775442cf899f267528918010 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/b2e095b5775442cf899f267528918010 2024-11-14T21:33:38,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/cd7dd8d49ff34e1c9e1d8ea3a58ac39f to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/cd7dd8d49ff34e1c9e1d8ea3a58ac39f 2024-11-14T21:33:38,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/2167cd5d7ff849d2893ff602957dd757 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/2167cd5d7ff849d2893ff602957dd757 2024-11-14T21:33:38,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/5ad8000ce9c64fd78cb6a33caed65e70 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/5ad8000ce9c64fd78cb6a33caed65e70 2024-11-14T21:33:38,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/be659a94510a4b3191baf0d362bdc2d6 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/be659a94510a4b3191baf0d362bdc2d6 2024-11-14T21:33:38,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/e23cadc43f3c4a8ab8799b11d67b40f1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/e23cadc43f3c4a8ab8799b11d67b40f1 2024-11-14T21:33:38,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a284a8f2e6b84dec9e30964c9f2e880e to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/a284a8f2e6b84dec9e30964c9f2e880e 2024-11-14T21:33:38,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/eab78ff6c5924cec8f2c413fe4badcb4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/eab78ff6c5924cec8f2c413fe4badcb4 2024-11-14T21:33:38,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/61d0a66564e74ccfad78bce40bc179bc to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/61d0a66564e74ccfad78bce40bc179bc 2024-11-14T21:33:38,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/db652e96748d434f811431ba3eb4f721 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/db652e96748d434f811431ba3eb4f721 2024-11-14T21:33:38,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/8191f793f2184ffda47c92bdecc174f5, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/8fca43e987984610a384b3820b474e31, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/bfef5965d5524572885c92e7097e2257, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/3feb8c2082374469973f7c86e0ac2c6e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/94bc0023ec2c4858a7cf77b1cb00f6d4, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/4f52ab1047084866bdec9c9e73660826, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/e1744d2d55e74600a0846d818f31bb9f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/940f1587ea2c45baa6e87eae859bc80c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/bf9824eaf74c4a37b336ab01fb73c67a, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/4d313a4dcf8d44e4892e0daca66db77b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/a0cc0a64e5ad46fc87a9917cd760d0db, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/7a283e0b2c5843b6a96beb6b55c80dec, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/bb2ed9522fd14047a77231eb156c856b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/d4bc0c49e3eb45c3a819d129c7a06bbb, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/03d6b5e435384ea283669e16544c9d8d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/b532eef1b28741de98f1041d79a793c2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/5ae9bd241e8a432bac30ad1f70397237, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/c8789e64f67a48ecbff66cd2cb0c37ee, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/47b27354a50145d0b7a7b3451b9e3fe0, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/48d53852b6144a3c8b46e8e483223c29, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/a58d9052cbc141dd88a8a987f9ed269c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/0326ff60302245f899ac4f730dffbb4c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/fabf0b3704b2411a86309a39862b0594, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/d396a17479df44f7ba38ec47cf423341, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/09fe2b423a9d4d00aff4a8a7cc850a1a, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/96b5bbf371ce433f90df612e3a9966da, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/de893e2bd0d74aa79efb6e8e06f79e46, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/a16ce2df481843f586baf29cca6af7ba, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/b24d684f22af4f9e9763e283042205d7, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/b7ff3588b92d481c91a12be69a6941bf, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/97a34f5997254a6aa51ca1c1c1f57a7a, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/cbed77af6702476da272dd7769f19e13, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/0da65d73bdbe4d498e18427590c69ecd, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/36e2088cfb3d4c6b98b6f918409470e4] to archive 2024-11-14T21:33:38,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:33:38,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/8191f793f2184ffda47c92bdecc174f5 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/8191f793f2184ffda47c92bdecc174f5 2024-11-14T21:33:38,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/8fca43e987984610a384b3820b474e31 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/8fca43e987984610a384b3820b474e31 2024-11-14T21:33:38,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/bfef5965d5524572885c92e7097e2257 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/bfef5965d5524572885c92e7097e2257 2024-11-14T21:33:38,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/3feb8c2082374469973f7c86e0ac2c6e to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/3feb8c2082374469973f7c86e0ac2c6e 2024-11-14T21:33:38,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/94bc0023ec2c4858a7cf77b1cb00f6d4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/94bc0023ec2c4858a7cf77b1cb00f6d4 2024-11-14T21:33:38,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/4f52ab1047084866bdec9c9e73660826 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/4f52ab1047084866bdec9c9e73660826 2024-11-14T21:33:38,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/e1744d2d55e74600a0846d818f31bb9f to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/e1744d2d55e74600a0846d818f31bb9f 2024-11-14T21:33:38,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/940f1587ea2c45baa6e87eae859bc80c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/940f1587ea2c45baa6e87eae859bc80c 2024-11-14T21:33:38,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/bf9824eaf74c4a37b336ab01fb73c67a to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/bf9824eaf74c4a37b336ab01fb73c67a 2024-11-14T21:33:38,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/4d313a4dcf8d44e4892e0daca66db77b to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/4d313a4dcf8d44e4892e0daca66db77b 2024-11-14T21:33:38,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/a0cc0a64e5ad46fc87a9917cd760d0db to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/a0cc0a64e5ad46fc87a9917cd760d0db 2024-11-14T21:33:38,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/7a283e0b2c5843b6a96beb6b55c80dec to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/7a283e0b2c5843b6a96beb6b55c80dec 2024-11-14T21:33:38,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/bb2ed9522fd14047a77231eb156c856b to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/bb2ed9522fd14047a77231eb156c856b 2024-11-14T21:33:38,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/d4bc0c49e3eb45c3a819d129c7a06bbb to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/d4bc0c49e3eb45c3a819d129c7a06bbb 2024-11-14T21:33:38,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/03d6b5e435384ea283669e16544c9d8d to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/03d6b5e435384ea283669e16544c9d8d 2024-11-14T21:33:38,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/b532eef1b28741de98f1041d79a793c2 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/b532eef1b28741de98f1041d79a793c2 2024-11-14T21:33:38,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/5ae9bd241e8a432bac30ad1f70397237 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/5ae9bd241e8a432bac30ad1f70397237 2024-11-14T21:33:38,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/c8789e64f67a48ecbff66cd2cb0c37ee to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/c8789e64f67a48ecbff66cd2cb0c37ee 2024-11-14T21:33:38,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/47b27354a50145d0b7a7b3451b9e3fe0 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/47b27354a50145d0b7a7b3451b9e3fe0 2024-11-14T21:33:38,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/48d53852b6144a3c8b46e8e483223c29 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/48d53852b6144a3c8b46e8e483223c29 2024-11-14T21:33:38,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/a58d9052cbc141dd88a8a987f9ed269c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/a58d9052cbc141dd88a8a987f9ed269c 2024-11-14T21:33:38,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/0326ff60302245f899ac4f730dffbb4c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/0326ff60302245f899ac4f730dffbb4c 2024-11-14T21:33:38,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/fabf0b3704b2411a86309a39862b0594 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/fabf0b3704b2411a86309a39862b0594 2024-11-14T21:33:38,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/d396a17479df44f7ba38ec47cf423341 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/d396a17479df44f7ba38ec47cf423341 2024-11-14T21:33:38,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/09fe2b423a9d4d00aff4a8a7cc850a1a to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/09fe2b423a9d4d00aff4a8a7cc850a1a 2024-11-14T21:33:38,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/96b5bbf371ce433f90df612e3a9966da to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/96b5bbf371ce433f90df612e3a9966da 2024-11-14T21:33:38,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/de893e2bd0d74aa79efb6e8e06f79e46 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/de893e2bd0d74aa79efb6e8e06f79e46 2024-11-14T21:33:38,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/a16ce2df481843f586baf29cca6af7ba to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/a16ce2df481843f586baf29cca6af7ba 2024-11-14T21:33:38,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/b24d684f22af4f9e9763e283042205d7 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/b24d684f22af4f9e9763e283042205d7 2024-11-14T21:33:38,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/b7ff3588b92d481c91a12be69a6941bf to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/b7ff3588b92d481c91a12be69a6941bf 2024-11-14T21:33:38,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/97a34f5997254a6aa51ca1c1c1f57a7a to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/97a34f5997254a6aa51ca1c1c1f57a7a 2024-11-14T21:33:38,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/cbed77af6702476da272dd7769f19e13 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/cbed77af6702476da272dd7769f19e13 2024-11-14T21:33:38,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/0da65d73bdbe4d498e18427590c69ecd to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/0da65d73bdbe4d498e18427590c69ecd 2024-11-14T21:33:38,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/36e2088cfb3d4c6b98b6f918409470e4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/36e2088cfb3d4c6b98b6f918409470e4 2024-11-14T21:33:38,599 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/recovered.edits/525.seqid, newMaxSeqId=525, maxSeqId=1 2024-11-14T21:33:38,600 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. 2024-11-14T21:33:38,600 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1676): Region close journal for 99a2a424d1830a6f519f1971fbc9bf41: Waiting for close lock at 1731620017270Running coprocessor pre-close hooks at 1731620017270Disabling compacts and flushes for region at 1731620017270Disabling writes for close at 1731620017270Obtaining lock to block concurrent updates at 1731620017270Preparing flush snapshotting stores in 99a2a424d1830a6f519f1971fbc9bf41 at 1731620017270Finished memstore snapshotting TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731620017271 (+1 ms)Flushing stores of TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41. at 1731620017272 (+1 ms)Flushing 99a2a424d1830a6f519f1971fbc9bf41/A: creating writer at 1731620017272Flushing 99a2a424d1830a6f519f1971fbc9bf41/A: appending metadata at 1731620017275 (+3 ms)Flushing 99a2a424d1830a6f519f1971fbc9bf41/A: closing flushed file at 1731620017275Flushing 99a2a424d1830a6f519f1971fbc9bf41/B: creating writer at 1731620017287 (+12 ms)Flushing 99a2a424d1830a6f519f1971fbc9bf41/B: appending metadata at 1731620017290 (+3 ms)Flushing 99a2a424d1830a6f519f1971fbc9bf41/B: closing flushed file at 1731620017290Flushing 99a2a424d1830a6f519f1971fbc9bf41/C: creating writer at 1731620017699 (+409 ms)Flushing 99a2a424d1830a6f519f1971fbc9bf41/C: appending metadata at 1731620017701 (+2 ms)Flushing 99a2a424d1830a6f519f1971fbc9bf41/C: closing flushed file at 1731620017701Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1dcab0ef: reopening flushed file at 1731620018140 (+439 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69435be4: reopening flushed file at 1731620018152 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1aabded3: reopening flushed file at 1731620018159 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 99a2a424d1830a6f519f1971fbc9bf41 in 903ms, sequenceid=522, compaction requested=true at 1731620018174 (+15 ms)Writing region close event to WAL at 1731620018595 (+421 ms)Running coprocessor post-close hooks at 1731620018600 (+5 ms)Closed at 1731620018600 2024-11-14T21:33:38,602 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] handler.UnassignRegionHandler(157): Closed 99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:38,603 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=99a2a424d1830a6f519f1971fbc9bf41, regionState=CLOSED 2024-11-14T21:33:38,605 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=72, state=RUNNABLE, hasLock=false; CloseRegionProcedure 99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 because future has completed 2024-11-14T21:33:38,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-14T21:33:38,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; CloseRegionProcedure 99a2a424d1830a6f519f1971fbc9bf41, server=ef788a4cade8,37683,1731619918560 in 1.4890 sec 2024-11-14T21:33:38,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=72, resume processing ppid=71 2024-11-14T21:33:38,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=71, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=99a2a424d1830a6f519f1971fbc9bf41, UNASSIGN in 1.4950 sec 2024-11-14T21:33:38,614 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-14T21:33:38,614 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4990 sec 2024-11-14T21:33:38,616 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731620018616"}]},"ts":"1731620018616"} 2024-11-14T21:33:38,617 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T21:33:38,617 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T21:33:38,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5130 sec 2024-11-14T21:33:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T21:33:39,262 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:33:39,262 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T21:33:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:33:39,265 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=74, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:33:39,266 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=74, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:33:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T21:33:39,273 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:39,294 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A, FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B, FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C, FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/recovered.edits] 2024-11-14T21:33:39,304 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/0fd15ea946cf44eba945924ce2e14df3 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/0fd15ea946cf44eba945924ce2e14df3 2024-11-14T21:33:39,307 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/63182b6de0314a398446549a1768cb63 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/63182b6de0314a398446549a1768cb63 2024-11-14T21:33:39,312 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/e3e033e8af9f4c76ac0c0ca0dd1c1f44 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/A/e3e033e8af9f4c76ac0c0ca0dd1c1f44 2024-11-14T21:33:39,323 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/1a60b9b1ff5543a48470107178e5041c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/1a60b9b1ff5543a48470107178e5041c 2024-11-14T21:33:39,331 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/28a123df488a4fd89e64924162cf2bcd to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/28a123df488a4fd89e64924162cf2bcd 2024-11-14T21:33:39,332 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/5321193cfb764764ae8c9128334d05c5 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/B/5321193cfb764764ae8c9128334d05c5 2024-11-14T21:33:39,337 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/2fd02b5252ad400c988bb044a2da10a9 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/2fd02b5252ad400c988bb044a2da10a9 2024-11-14T21:33:39,343 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/3a7ec5b51e5849048557ca243433f545 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/3a7ec5b51e5849048557ca243433f545 2024-11-14T21:33:39,345 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/927fe9bb565c4a7d9c1bb80ab9ff366f to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/C/927fe9bb565c4a7d9c1bb80ab9ff366f 2024-11-14T21:33:39,355 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/recovered.edits/525.seqid to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41/recovered.edits/525.seqid 2024-11-14T21:33:39,355 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/99a2a424d1830a6f519f1971fbc9bf41 2024-11-14T21:33:39,355 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T21:33:39,358 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=74, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:33:39,362 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T21:33:39,364 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T21:33:39,366 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=74, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:33:39,366 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T21:33:39,366 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731620019366"}]},"ts":"9223372036854775807"} 2024-11-14T21:33:39,368 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T21:33:39,368 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 99a2a424d1830a6f519f1971fbc9bf41, NAME => 'TestAcidGuarantees,,1731619986714.99a2a424d1830a6f519f1971fbc9bf41.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T21:33:39,369 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T21:33:39,369 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731620019369"}]},"ts":"9223372036854775807"} 2024-11-14T21:33:39,371 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T21:33:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T21:33:39,372 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=74, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:33:39,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 110 msec 2024-11-14T21:33:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T21:33:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T21:33:39,581 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:33:39,590 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 249), OpenFileDescriptor=449 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=957 (was 902) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1231 (was 882) - AvailableMemoryMB LEAK? - 2024-11-14T21:33:39,599 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=957, ProcessCount=11, AvailableMemoryMB=1230 2024-11-14T21:33:39,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T21:33:39,600 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T21:33:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T21:33:39,602 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T21:33:39,602 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:39,602 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 75 2024-11-14T21:33:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-11-14T21:33:39,603 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T21:33:39,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742155_1331 (size=1061) 2024-11-14T21:33:39,610 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275 2024-11-14T21:33:39,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742156_1332 (size=53) 2024-11-14T21:33:39,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-11-14T21:33:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-11-14T21:33:40,016 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:33:40,016 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c47d7036b7faae0ba1cfffe0348781b1, disabling compactions & flushes 2024-11-14T21:33:40,016 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:40,016 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:40,016 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. after waiting 0 ms 2024-11-14T21:33:40,016 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:40,016 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:40,016 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c47d7036b7faae0ba1cfffe0348781b1: Waiting for close lock at 1731620020016Disabling compacts and flushes for region at 1731620020016Disabling writes for close at 1731620020016Writing region close event to WAL at 1731620020016Closed at 1731620020016 2024-11-14T21:33:40,020 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T21:33:40,020 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731620020020"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731620020020"}]},"ts":"1731620020020"} 2024-11-14T21:33:40,022 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T21:33:40,023 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T21:33:40,023 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731620020023"}]},"ts":"1731620020023"} 2024-11-14T21:33:40,032 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T21:33:40,033 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c47d7036b7faae0ba1cfffe0348781b1, ASSIGN}] 2024-11-14T21:33:40,034 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c47d7036b7faae0ba1cfffe0348781b1, ASSIGN 2024-11-14T21:33:40,035 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c47d7036b7faae0ba1cfffe0348781b1, ASSIGN; state=OFFLINE, location=ef788a4cade8,37683,1731619918560; forceNewPlan=false, retain=false 2024-11-14T21:33:40,187 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=c47d7036b7faae0ba1cfffe0348781b1, regionState=OPENING, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:40,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c47d7036b7faae0ba1cfffe0348781b1, ASSIGN because future has completed 2024-11-14T21:33:40,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; OpenRegionProcedure c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560}] 2024-11-14T21:33:40,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-11-14T21:33:40,375 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:40,375 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:33:40,376 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:40,376 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:33:40,376 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:40,376 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:40,391 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:40,399 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:33:40,399 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c47d7036b7faae0ba1cfffe0348781b1 columnFamilyName A 2024-11-14T21:33:40,399 DEBUG [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:40,400 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] regionserver.HStore(327): Store=c47d7036b7faae0ba1cfffe0348781b1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:33:40,400 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:40,402 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:33:40,402 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c47d7036b7faae0ba1cfffe0348781b1 columnFamilyName B 2024-11-14T21:33:40,402 DEBUG [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:40,403 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] regionserver.HStore(327): Store=c47d7036b7faae0ba1cfffe0348781b1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:33:40,403 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:40,404 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:33:40,405 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c47d7036b7faae0ba1cfffe0348781b1 columnFamilyName C 2024-11-14T21:33:40,405 DEBUG [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:40,405 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] regionserver.HStore(327): Store=c47d7036b7faae0ba1cfffe0348781b1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:33:40,405 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:40,405 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:40,407 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:40,407 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:40,408 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:40,409 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:40,409 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T21:33:40,419 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:40,427 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T21:33:40,427 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened c47d7036b7faae0ba1cfffe0348781b1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64505535, jitterRate=-0.038792625069618225}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T21:33:40,427 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:40,428 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for c47d7036b7faae0ba1cfffe0348781b1: Running coprocessor pre-open hook at 1731620020376Writing region info on filesystem at 1731620020376Initializing all the Stores at 1731620020387 (+11 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731620020387Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731620020390 (+3 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731620020390Cleaning up temporary data from old regions at 1731620020409 (+19 ms)Running coprocessor post-open hooks at 1731620020427 (+18 ms)Region opened successfully at 1731620020428 (+1 ms) 2024-11-14T21:33:40,429 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., pid=77, masterSystemTime=1731620020363 2024-11-14T21:33:40,433 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=c47d7036b7faae0ba1cfffe0348781b1, regionState=OPEN, openSeqNum=2, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:40,438 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:40,439 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:40,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=76, state=RUNNABLE, hasLock=false; OpenRegionProcedure c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 because future has completed 2024-11-14T21:33:40,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-14T21:33:40,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; OpenRegionProcedure c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 in 232 msec 2024-11-14T21:33:40,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75 2024-11-14T21:33:40,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c47d7036b7faae0ba1cfffe0348781b1, ASSIGN in 417 msec 2024-11-14T21:33:40,457 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T21:33:40,457 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731620020457"}]},"ts":"1731620020457"} 2024-11-14T21:33:40,461 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T21:33:40,463 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T21:33:40,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 863 msec 2024-11-14T21:33:40,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-11-14T21:33:40,743 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:33:40,749 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e48fd67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:40,749 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:33:40,749 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:33:40,750 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:33:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:33:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:33:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@323fa567, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:40,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:33:40,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:33:40,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:40,752 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:33:40,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ede2b05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:40,753 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:33:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:33:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:33:40,754 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:33:40,755 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T21:33:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is ef788a4cade8,38255,1731619917503 2024-11-14T21:33:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@73cccc71 2024-11-14T21:33:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T21:33:40,756 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T21:33:40,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T21:33:40,758 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T21:33:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T21:33:40,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742157_1333 (size=1097) 2024-11-14T21:33:41,182 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-14T21:33:41,182 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-14T21:33:41,185 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T21:33:41,188 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c47d7036b7faae0ba1cfffe0348781b1, REOPEN/MOVE}] 2024-11-14T21:33:41,189 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c47d7036b7faae0ba1cfffe0348781b1, REOPEN/MOVE 2024-11-14T21:33:41,191 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=c47d7036b7faae0ba1cfffe0348781b1, regionState=CLOSING, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:41,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c47d7036b7faae0ba1cfffe0348781b1, REOPEN/MOVE because future has completed 2024-11-14T21:33:41,192 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T21:33:41,192 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560}] 2024-11-14T21:33:41,345 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:41,345 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T21:33:41,345 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing c47d7036b7faae0ba1cfffe0348781b1, disabling compactions & flushes 2024-11-14T21:33:41,345 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:41,345 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:41,345 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. after waiting 0 ms 2024-11-14T21:33:41,345 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:41,355 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T21:33:41,356 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:41,356 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for c47d7036b7faae0ba1cfffe0348781b1: Waiting for close lock at 1731620021345Running coprocessor pre-close hooks at 1731620021345Disabling compacts and flushes for region at 1731620021345Disabling writes for close at 1731620021345Writing region close event to WAL at 1731620021349 (+4 ms)Running coprocessor post-close hooks at 1731620021356 (+7 ms)Closed at 1731620021356 2024-11-14T21:33:41,356 WARN [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionServer(3299): Not adding moved region record: c47d7036b7faae0ba1cfffe0348781b1 to self. 2024-11-14T21:33:41,358 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:41,359 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=c47d7036b7faae0ba1cfffe0348781b1, regionState=CLOSED 2024-11-14T21:33:41,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 because future has completed 2024-11-14T21:33:41,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-14T21:33:41,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 in 171 msec 2024-11-14T21:33:41,366 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c47d7036b7faae0ba1cfffe0348781b1, REOPEN/MOVE; state=CLOSED, location=ef788a4cade8,37683,1731619918560; forceNewPlan=false, retain=true 2024-11-14T21:33:41,516 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=c47d7036b7faae0ba1cfffe0348781b1, regionState=OPENING, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:41,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c47d7036b7faae0ba1cfffe0348781b1, REOPEN/MOVE because future has completed 2024-11-14T21:33:41,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560}] 2024-11-14T21:33:41,680 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:41,680 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:33:41,680 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:41,680 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:33:41,680 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:41,680 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:41,691 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:41,699 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:33:41,700 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c47d7036b7faae0ba1cfffe0348781b1 columnFamilyName A 2024-11-14T21:33:41,702 DEBUG [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:41,703 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] regionserver.HStore(327): Store=c47d7036b7faae0ba1cfffe0348781b1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:33:41,704 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:41,705 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:33:41,705 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c47d7036b7faae0ba1cfffe0348781b1 columnFamilyName B 2024-11-14T21:33:41,705 DEBUG [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:41,715 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] regionserver.HStore(327): Store=c47d7036b7faae0ba1cfffe0348781b1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:33:41,715 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:41,723 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:33:41,723 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c47d7036b7faae0ba1cfffe0348781b1 columnFamilyName C 2024-11-14T21:33:41,723 DEBUG [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:41,726 INFO [StoreOpener-c47d7036b7faae0ba1cfffe0348781b1-1 {}] regionserver.HStore(327): Store=c47d7036b7faae0ba1cfffe0348781b1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:33:41,727 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:41,727 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:41,727 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:41,735 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:41,736 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:41,736 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:41,737 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T21:33:41,738 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:41,739 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened c47d7036b7faae0ba1cfffe0348781b1; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72710116, jitterRate=0.08346515893936157}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T21:33:41,739 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:41,743 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for c47d7036b7faae0ba1cfffe0348781b1: Running coprocessor pre-open hook at 1731620021680Writing region info on filesystem at 1731620021680Initializing all the Stores at 1731620021691 (+11 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731620021691Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731620021691Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731620021691Cleaning up temporary data from old regions at 1731620021736 (+45 ms)Running coprocessor post-open hooks at 1731620021739 (+3 ms)Region opened successfully at 1731620021740 (+1 ms) 2024-11-14T21:33:41,744 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., pid=82, masterSystemTime=1731620021671 2024-11-14T21:33:41,751 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:41,751 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:41,755 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=c47d7036b7faae0ba1cfffe0348781b1, regionState=OPEN, openSeqNum=5, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:41,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 because future has completed 2024-11-14T21:33:41,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=80 2024-11-14T21:33:41,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=80, state=SUCCESS, hasLock=false; OpenRegionProcedure c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 in 243 msec 2024-11-14T21:33:41,767 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-14T21:33:41,767 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c47d7036b7faae0ba1cfffe0348781b1, REOPEN/MOVE in 577 msec 2024-11-14T21:33:41,770 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-14T21:33:41,770 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 583 msec 2024-11-14T21:33:41,773 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0130 sec 2024-11-14T21:33:41,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T21:33:41,774 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:33:41,775 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@662d67c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,775 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:33:41,776 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:33:41,779 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:33:41,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:33:41,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:33:41,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f93911, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:33:41,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:33:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:41,782 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:33:41,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699691a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6312c29f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:33:41,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:33:41,794 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:33:41,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:33:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:33:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f68fd09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:33:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:33:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:41,798 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:33:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a077d04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a50ed2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,804 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:33:41,804 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:33:41,815 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:33:41,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:33:41,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:33:41,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b961602, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:33:41,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:33:41,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:41,825 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:33:41,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a47fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,831 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16985d76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,831 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:33:41,831 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:33:41,833 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:33:41,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:33:41,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:33:41,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25691a89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:33:41,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:33:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:41,836 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:33:41,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45f205dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d71b451, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,855 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:33:41,855 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:33:41,856 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:33:41,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:33:41,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:33:41,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36c49bfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:33:41,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:33:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:41,860 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:33:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43419f5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ea2dc71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,862 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:33:41,862 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:33:41,864 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:33:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:33:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:33:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c1b0167, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:33:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:33:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:41,866 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:33:41,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28499bc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,879 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c218ee1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,879 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:33:41,879 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:33:41,885 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:33:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:33:41,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:33:41,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d4a0c8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:33:41,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:33:41,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:41,901 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:33:41,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a3c5b8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47eafbc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,906 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:33:41,906 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:33:41,916 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:33:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:33:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:33:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@725a001a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:33:41,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:33:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:41,921 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:33:41,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4559db0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,939 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b92b23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,939 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:33:41,939 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:33:41,941 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:33:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:33:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:33:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3def8f2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:33:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:33:41,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:41,944 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:33:41,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f8fcd2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af44064, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,947 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:33:41,947 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:33:41,948 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:33:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:33:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:33:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f48cd02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:33:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:33:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:33:41,950 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:33:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@484b1377, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:33:41,951 DEBUG [Thread-1531 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:33:41,952 DEBUG [Thread-1531 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:33:41,953 DEBUG [Thread-1531 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:33:41,954 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:33:41,956 DEBUG [Thread-1531 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:33:41,961 DEBUG [Thread-1542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:33:41,962 DEBUG [Thread-1533 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:33:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:33:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:33:41,964 DEBUG [Thread-1542 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:33:41,964 DEBUG [Thread-1542 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:33:41,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:33:41,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:33:41,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:33:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:33:41,970 DEBUG [Thread-1527 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:33:41,970 DEBUG [Thread-1538 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:33:41,971 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:33:41,971 DEBUG [Thread-1540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:33:41,971 DEBUG [Thread-1527 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:33:41,972 DEBUG [Thread-1527 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:33:41,972 DEBUG [Thread-1540 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:33:41,972 DEBUG [Thread-1540 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:33:41,973 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:33:41,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees 2024-11-14T21:33:41,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-14T21:33:41,975 DEBUG [Thread-1525 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:33:41,975 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=83, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:33:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:33:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:33:41,976 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=83, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:33:41,976 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:33:41,977 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:33:41,977 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:33:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:33:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:33:41,983 DEBUG [Thread-1536 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:33:41,983 DEBUG [Thread-1527 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:33:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:41,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c47d7036b7faae0ba1cfffe0348781b1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T21:33:41,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:33:41,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:33:41,987 DEBUG [Thread-1544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:33:41,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:33:41,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:33:41,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=A 2024-11-14T21:33:41,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:41,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=B 2024-11-14T21:33:41,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:41,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=C 2024-11-14T21:33:41,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:41,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:33:41,993 DEBUG [Thread-1529 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:33:41,993 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:33:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:33:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:33:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:33:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:33:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:33:41,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:33:42,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:33:42,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:33:42,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:33:42,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:33:42,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:42,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620082036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:42,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:42,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620082037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:42,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:42,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:42,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620082052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:42,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:42,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620082053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:42,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:42,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:42,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a67cd46904d0490895405d0b89211be4_c47d7036b7faae0ba1cfffe0348781b1 is 50, key is test_row_0/A:col10/1731620021973/Put/seqid=0 2024-11-14T21:33:42,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620082060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-14T21:33:42,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742158_1334 (size=12154) 2024-11-14T21:33:42,100 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:42,111 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a67cd46904d0490895405d0b89211be4_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a67cd46904d0490895405d0b89211be4_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:42,114 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/2bc45d594cd9436c8d5dec600893b948, store: [table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:42,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/2bc45d594cd9436c8d5dec600893b948 is 175, key is test_row_0/A:col10/1731620021973/Put/seqid=0 2024-11-14T21:33:42,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-14T21:33:42,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:42,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:42,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:42,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:42,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:42,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:42,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742159_1335 (size=30955) 2024-11-14T21:33:42,156 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/2bc45d594cd9436c8d5dec600893b948 2024-11-14T21:33:42,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620082219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:42,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:42,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620082219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620082219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:42,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620082219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:42,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:42,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:42,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:42,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620082222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:42,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/762cd60921b64d95a667d8f6fb4ab443 is 50, key is test_row_0/B:col10/1731620021973/Put/seqid=0 2024-11-14T21:33:42,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-14T21:33:42,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:42,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:42,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:42,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:42,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:42,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:42,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-14T21:33:42,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742160_1336 (size=12001) 2024-11-14T21:33:42,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/762cd60921b64d95a667d8f6fb4ab443 2024-11-14T21:33:42,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/795ecb3d8808428fbb09b6a9d498419d is 50, key is test_row_0/C:col10/1731620021973/Put/seqid=0 2024-11-14T21:33:42,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-14T21:33:42,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:42,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:42,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:42,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:42,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:42,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620082432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620082432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:42,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620082433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:42,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620082433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:42,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:42,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:42,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620082443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:42,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742161_1337 (size=12001) 2024-11-14T21:33:42,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-14T21:33:42,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-14T21:33:42,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:42,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:42,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:42,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:42,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:42,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-14T21:33:42,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:42,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:42,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:42,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:42,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:42,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:42,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620082761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:42,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620082761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:42,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620082761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620082770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:42,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:42,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:42,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620082774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:42,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:42,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:42,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/795ecb3d8808428fbb09b6a9d498419d 2024-11-14T21:33:42,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/2bc45d594cd9436c8d5dec600893b948 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/2bc45d594cd9436c8d5dec600893b948 2024-11-14T21:33:42,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/2bc45d594cd9436c8d5dec600893b948, entries=150, sequenceid=16, filesize=30.2 K 2024-11-14T21:33:42,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/762cd60921b64d95a667d8f6fb4ab443 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/762cd60921b64d95a667d8f6fb4ab443 2024-11-14T21:33:42,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/762cd60921b64d95a667d8f6fb4ab443, entries=150, sequenceid=16, filesize=11.7 K 2024-11-14T21:33:42,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/795ecb3d8808428fbb09b6a9d498419d as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/795ecb3d8808428fbb09b6a9d498419d 2024-11-14T21:33:42,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-14T21:33:42,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:42,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:42,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:42,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:42,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:42,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:42,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/795ecb3d8808428fbb09b6a9d498419d, entries=150, sequenceid=16, filesize=11.7 K 2024-11-14T21:33:42,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c47d7036b7faae0ba1cfffe0348781b1 in 980ms, sequenceid=16, compaction requested=false 2024-11-14T21:33:42,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:43,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-14T21:33:43,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:43,086 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2902): Flushing c47d7036b7faae0ba1cfffe0348781b1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T21:33:43,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=A 2024-11-14T21:33:43,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:43,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=B 2024-11-14T21:33:43,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:43,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=C 2024-11-14T21:33:43,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:43,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-14T21:33:43,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114565e621e63da48aea09eca75c5510cd9_c47d7036b7faae0ba1cfffe0348781b1 is 50, key is test_row_0/A:col10/1731620022029/Put/seqid=0 2024-11-14T21:33:43,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742162_1338 (size=12154) 2024-11-14T21:33:43,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,193 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114565e621e63da48aea09eca75c5510cd9_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114565e621e63da48aea09eca75c5510cd9_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:43,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/50eace1131ea4973ae9700b10ed2f41e, store: [table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:43,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/50eace1131ea4973ae9700b10ed2f41e is 175, key is test_row_0/A:col10/1731620022029/Put/seqid=0 2024-11-14T21:33:43,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742163_1339 (size=30955) 2024-11-14T21:33:43,263 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/50eace1131ea4973ae9700b10ed2f41e 2024-11-14T21:33:43,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/9bab9952a7634cef90dfd7ec8d311f76 is 50, key is test_row_0/B:col10/1731620022029/Put/seqid=0 2024-11-14T21:33:43,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:43,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:43,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742164_1340 (size=12001) 2024-11-14T21:33:43,325 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/9bab9952a7634cef90dfd7ec8d311f76 2024-11-14T21:33:43,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/a21ee0eafcae4050a49997f913e1a4e1 is 50, key is test_row_0/C:col10/1731620022029/Put/seqid=0 2024-11-14T21:33:43,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:43,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620083319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:43,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:43,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:43,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620083337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:43,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:43,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:43,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620083359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:43,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:43,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620083360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:43,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:43,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:43,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620083364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:43,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:43,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742165_1341 (size=12001) 2024-11-14T21:33:43,417 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/a21ee0eafcae4050a49997f913e1a4e1 2024-11-14T21:33:43,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/50eace1131ea4973ae9700b10ed2f41e as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/50eace1131ea4973ae9700b10ed2f41e 2024-11-14T21:33:43,440 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/50eace1131ea4973ae9700b10ed2f41e, entries=150, sequenceid=36, filesize=30.2 K 2024-11-14T21:33:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/9bab9952a7634cef90dfd7ec8d311f76 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/9bab9952a7634cef90dfd7ec8d311f76 2024-11-14T21:33:43,467 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/9bab9952a7634cef90dfd7ec8d311f76, entries=150, sequenceid=36, filesize=11.7 K 2024-11-14T21:33:43,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/a21ee0eafcae4050a49997f913e1a4e1 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/a21ee0eafcae4050a49997f913e1a4e1 2024-11-14T21:33:43,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,480 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/a21ee0eafcae4050a49997f913e1a4e1, entries=150, sequenceid=36, filesize=11.7 K 2024-11-14T21:33:43,482 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c47d7036b7faae0ba1cfffe0348781b1 in 395ms, sequenceid=36, compaction requested=false 2024-11-14T21:33:43,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2603): Flush status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:43,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:43,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=84 2024-11-14T21:33:43,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=84 2024-11-14T21:33:43,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,512 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-14T21:33:43,512 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5300 sec 2024-11-14T21:33:43,514 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees in 1.5420 sec 2024-11-14T21:33:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:43,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c47d7036b7faae0ba1cfffe0348781b1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T21:33:43,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=A 2024-11-14T21:33:43,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:43,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=B 2024-11-14T21:33:43,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:43,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=C 2024-11-14T21:33:43,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114133a7830917a4956a4555c90a2bdef4a_c47d7036b7faae0ba1cfffe0348781b1 is 50, key is test_row_0/A:col10/1731620023313/Put/seqid=0 2024-11-14T21:33:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742166_1342 (size=12154) 2024-11-14T21:33:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,730 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:43,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620083697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:43,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620083713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620083715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:43,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620083719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620083731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:43,749 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114133a7830917a4956a4555c90a2bdef4a_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114133a7830917a4956a4555c90a2bdef4a_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:43,753 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/df0c14b241e64664bc00b7de147ba207, store: [table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:43,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/df0c14b241e64664bc00b7de147ba207 is 175, key is test_row_0/A:col10/1731620023313/Put/seqid=0 2024-11-14T21:33:43,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742167_1343 (size=30955) 2024-11-14T21:33:43,783 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/df0c14b241e64664bc00b7de147ba207 2024-11-14T21:33:43,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/35ef74964131426097183d5ed3c870b3 is 50, key is test_row_0/B:col10/1731620023313/Put/seqid=0 2024-11-14T21:33:43,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742168_1344 (size=12001) 2024-11-14T21:33:43,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620083841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:43,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:43,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620083843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:43,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:43,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620083843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:43,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:43,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620083850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:43,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:43,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620083850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:43,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:43,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:43,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:44,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620084060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:44,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620084060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:44,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620084060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:44,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620084071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:44,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:44,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620084071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:44,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:44,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-14T21:33:44,121 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:33:44,122 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:33:44,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees 2024-11-14T21:33:44,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T21:33:44,125 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:33:44,126 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:33:44,126 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:33:44,151 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T21:33:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T21:33:44,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/35ef74964131426097183d5ed3c870b3 2024-11-14T21:33:44,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-14T21:33:44,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:44,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:44,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:44,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:44,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:44,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/e4d2d0b18ff64cf18c9621b264549460 is 50, key is test_row_0/C:col10/1731620023313/Put/seqid=0 2024-11-14T21:33:44,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742169_1345 (size=12001) 2024-11-14T21:33:44,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/e4d2d0b18ff64cf18c9621b264549460 2024-11-14T21:33:44,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/df0c14b241e64664bc00b7de147ba207 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/df0c14b241e64664bc00b7de147ba207 2024-11-14T21:33:44,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/df0c14b241e64664bc00b7de147ba207, entries=150, sequenceid=49, filesize=30.2 K 2024-11-14T21:33:44,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/35ef74964131426097183d5ed3c870b3 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/35ef74964131426097183d5ed3c870b3 2024-11-14T21:33:44,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/35ef74964131426097183d5ed3c870b3, entries=150, sequenceid=49, filesize=11.7 K 2024-11-14T21:33:44,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/e4d2d0b18ff64cf18c9621b264549460 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/e4d2d0b18ff64cf18c9621b264549460 2024-11-14T21:33:44,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/e4d2d0b18ff64cf18c9621b264549460, entries=150, sequenceid=49, filesize=11.7 K 2024-11-14T21:33:44,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c47d7036b7faae0ba1cfffe0348781b1 in 737ms, sequenceid=49, compaction requested=true 2024-11-14T21:33:44,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:44,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:33:44,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:44,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:33:44,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:44,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:33:44,365 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:44,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:33:44,367 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:44,367 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:44,367 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/A is initiating minor compaction (all files) 2024-11-14T21:33:44,367 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/A in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:44,367 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/2bc45d594cd9436c8d5dec600893b948, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/50eace1131ea4973ae9700b10ed2f41e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/df0c14b241e64664bc00b7de147ba207] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=90.7 K 2024-11-14T21:33:44,367 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:44,367 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. files: [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/2bc45d594cd9436c8d5dec600893b948, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/50eace1131ea4973ae9700b10ed2f41e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/df0c14b241e64664bc00b7de147ba207] 2024-11-14T21:33:44,367 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:44,368 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/B is initiating minor compaction (all files) 2024-11-14T21:33:44,368 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/B in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:44,368 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/762cd60921b64d95a667d8f6fb4ab443, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/9bab9952a7634cef90dfd7ec8d311f76, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/35ef74964131426097183d5ed3c870b3] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=35.2 K 2024-11-14T21:33:44,368 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 762cd60921b64d95a667d8f6fb4ab443, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731620021961 2024-11-14T21:33:44,368 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bab9952a7634cef90dfd7ec8d311f76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731620022029 2024-11-14T21:33:44,369 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 35ef74964131426097183d5ed3c870b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731620023313 2024-11-14T21:33:44,369 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bc45d594cd9436c8d5dec600893b948, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731620021961 2024-11-14T21:33:44,370 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50eace1131ea4973ae9700b10ed2f41e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731620022029 2024-11-14T21:33:44,370 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting df0c14b241e64664bc00b7de147ba207, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731620023313 2024-11-14T21:33:44,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c47d7036b7faae0ba1cfffe0348781b1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T21:33:44,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=A 2024-11-14T21:33:44,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:44,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=B 2024-11-14T21:33:44,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:44,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=C 2024-11-14T21:33:44,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:44,385 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#B#compaction#299 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:44,386 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/00ce6fe3e6c0436185946eea3a67aed8 is 50, key is test_row_0/B:col10/1731620023313/Put/seqid=0 2024-11-14T21:33:44,393 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:44,410 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114ae45e12353634625a33d53a63e14d96e_c47d7036b7faae0ba1cfffe0348781b1 store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:44,412 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114ae45e12353634625a33d53a63e14d96e_c47d7036b7faae0ba1cfffe0348781b1, store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:44,412 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ae45e12353634625a33d53a63e14d96e_c47d7036b7faae0ba1cfffe0348781b1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:44,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742170_1346 (size=12104) 2024-11-14T21:33:44,432 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/00ce6fe3e6c0436185946eea3a67aed8 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/00ce6fe3e6c0436185946eea3a67aed8 2024-11-14T21:33:44,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148226e5cc03174ed1af84943bf15f6a17_c47d7036b7faae0ba1cfffe0348781b1 is 50, key is test_row_0/A:col10/1731620024379/Put/seqid=0 2024-11-14T21:33:44,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:44,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-14T21:33:44,442 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/B of c47d7036b7faae0ba1cfffe0348781b1 into 00ce6fe3e6c0436185946eea3a67aed8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:44,443 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:44,443 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/B, priority=13, startTime=1731620024365; duration=0sec 2024-11-14T21:33:44,443 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:44,443 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:B 2024-11-14T21:33:44,443 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:44,444 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:44,444 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/C is initiating minor compaction (all files) 2024-11-14T21:33:44,444 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/C in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:44,444 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/795ecb3d8808428fbb09b6a9d498419d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/a21ee0eafcae4050a49997f913e1a4e1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/e4d2d0b18ff64cf18c9621b264549460] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=35.2 K 2024-11-14T21:33:44,445 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 795ecb3d8808428fbb09b6a9d498419d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731620021961 2024-11-14T21:33:44,446 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting a21ee0eafcae4050a49997f913e1a4e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731620022029 2024-11-14T21:33:44,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:44,446 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting e4d2d0b18ff64cf18c9621b264549460, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731620023313 2024-11-14T21:33:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620084412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620084410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:44,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:44,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:44,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:44,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:44,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:44,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:44,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:44,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:44,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620084416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:44,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T21:33:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620084418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:44,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620084419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:44,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:44,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742171_1347 (size=4469) 2024-11-14T21:33:44,470 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#A#compaction#300 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:44,471 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/dcb4d04957aa45dd851d316c885db620 is 175, key is test_row_0/A:col10/1731620023313/Put/seqid=0 2024-11-14T21:33:44,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742172_1348 (size=14594) 2024-11-14T21:33:44,495 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#C#compaction#302 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:44,495 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/cf78ff02093e49a383d8505ba7a2b583 is 50, key is test_row_0/C:col10/1731620023313/Put/seqid=0 2024-11-14T21:33:44,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742174_1350 (size=12104) 2024-11-14T21:33:44,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742173_1349 (size=31058) 2024-11-14T21:33:44,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620084551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:44,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:44,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620084554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:44,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620084561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:44,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620084561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:44,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620084563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:44,596 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/cf78ff02093e49a383d8505ba7a2b583 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/cf78ff02093e49a383d8505ba7a2b583 2024-11-14T21:33:44,602 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/C of c47d7036b7faae0ba1cfffe0348781b1 into cf78ff02093e49a383d8505ba7a2b583(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:44,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-14T21:33:44,602 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:44,602 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/C, priority=13, startTime=1731620024365; duration=0sec 2024-11-14T21:33:44,602 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:44,602 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:C 2024-11-14T21:33:44,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:44,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:44,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:44,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:44,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:44,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-14T21:33:44,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:44,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:44,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:44,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:44,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:44,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T21:33:44,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620084799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:44,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620084806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:44,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620084806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:44,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:44,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:44,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620084806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:44,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:44,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620084806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:44,894 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:44,923 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148226e5cc03174ed1af84943bf15f6a17_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148226e5cc03174ed1af84943bf15f6a17_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:44,924 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/c7f88ae6aa05447b9ec27eeaee3e8193, store: [table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:44,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/c7f88ae6aa05447b9ec27eeaee3e8193 is 175, key is test_row_0/A:col10/1731620024379/Put/seqid=0 2024-11-14T21:33:44,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-14T21:33:44,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:44,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:44,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:44,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:44,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:44,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:44,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742175_1351 (size=39549) 2024-11-14T21:33:44,995 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/dcb4d04957aa45dd851d316c885db620 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/dcb4d04957aa45dd851d316c885db620 2024-11-14T21:33:45,042 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/A of c47d7036b7faae0ba1cfffe0348781b1 into dcb4d04957aa45dd851d316c885db620(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:45,042 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:45,042 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/A, priority=13, startTime=1731620024365; duration=0sec 2024-11-14T21:33:45,042 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:45,042 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:A 2024-11-14T21:33:45,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-14T21:33:45,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:45,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:45,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:45,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:45,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:45,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620085121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:45,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620085121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:45,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:45,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620085131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:45,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:45,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:45,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620085131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:45,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620085131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:45,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-14T21:33:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:45,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:45,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:45,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T21:33:45,384 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=69, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/c7f88ae6aa05447b9ec27eeaee3e8193 2024-11-14T21:33:45,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-14T21:33:45,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:45,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:45,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:45,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:45,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:45,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/133c3a15ff7c482093ec134fd4e19b3e is 50, key is test_row_0/B:col10/1731620024379/Put/seqid=0 2024-11-14T21:33:45,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742176_1352 (size=12001) 2024-11-14T21:33:45,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/133c3a15ff7c482093ec134fd4e19b3e 2024-11-14T21:33:45,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/2fdce2cb2fea4d46803ad70ddc682157 is 50, key is test_row_0/C:col10/1731620024379/Put/seqid=0 2024-11-14T21:33:45,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742177_1353 (size=12001) 2024-11-14T21:33:45,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/2fdce2cb2fea4d46803ad70ddc682157 2024-11-14T21:33:45,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/c7f88ae6aa05447b9ec27eeaee3e8193 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/c7f88ae6aa05447b9ec27eeaee3e8193 2024-11-14T21:33:45,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-14T21:33:45,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/c7f88ae6aa05447b9ec27eeaee3e8193, entries=200, sequenceid=69, filesize=38.6 K 2024-11-14T21:33:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:45,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:45,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:45,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/133c3a15ff7c482093ec134fd4e19b3e as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/133c3a15ff7c482093ec134fd4e19b3e 2024-11-14T21:33:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:45,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/133c3a15ff7c482093ec134fd4e19b3e, entries=150, sequenceid=69, filesize=11.7 K 2024-11-14T21:33:45,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/2fdce2cb2fea4d46803ad70ddc682157 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/2fdce2cb2fea4d46803ad70ddc682157 2024-11-14T21:33:45,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/2fdce2cb2fea4d46803ad70ddc682157, entries=150, sequenceid=69, filesize=11.7 K 2024-11-14T21:33:45,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c47d7036b7faae0ba1cfffe0348781b1 in 1209ms, sequenceid=69, compaction requested=false 2024-11-14T21:33:45,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:45,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c47d7036b7faae0ba1cfffe0348781b1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T21:33:45,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=A 2024-11-14T21:33:45,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:45,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=B 2024-11-14T21:33:45,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:45,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=C 2024-11-14T21:33:45,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:45,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145ac9e520c2e8431c8f3755948907b3ed_c47d7036b7faae0ba1cfffe0348781b1 is 50, key is test_row_0/A:col10/1731620024410/Put/seqid=0 2024-11-14T21:33:45,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-14T21:33:45,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:45,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:45,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:45,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:45,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:45,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742178_1354 (size=12154) 2024-11-14T21:33:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:45,744 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:45,765 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145ac9e520c2e8431c8f3755948907b3ed_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145ac9e520c2e8431c8f3755948907b3ed_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:45,767 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/db941d86545e4188893d73292fbae917, store: [table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:45,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/db941d86545e4188893d73292fbae917 is 175, key is test_row_0/A:col10/1731620024410/Put/seqid=0 2024-11-14T21:33:45,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620085758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:45,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620085765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:45,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:45,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742179_1355 (size=30955) 2024-11-14T21:33:45,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:45,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620085768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:45,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:45,834 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/db941d86545e4188893d73292fbae917 2024-11-14T21:33:45,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620085817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:45,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:45,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620085817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:45,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:45,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/d9aefa8d9bef408493f1cf4a523af04d is 50, key is test_row_0/B:col10/1731620024410/Put/seqid=0 2024-11-14T21:33:45,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-14T21:33:45,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:45,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:45,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:45,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:45,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:45,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620085921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:45,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742180_1356 (size=12001) 2024-11-14T21:33:45,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620085930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:45,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620085930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:45,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:45,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620085971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:45,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:45,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:46,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620085980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:46,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:46,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-14T21:33:46,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:46,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:46,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:46,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:46,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:46,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620086147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:46,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620086150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:46,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:46,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620086158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:46,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:46,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:46,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:46,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:46,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:46,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-14T21:33:46,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:46,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:46,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:46,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:46,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:46,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:46,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620086211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:46,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620086211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T21:33:46,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/d9aefa8d9bef408493f1cf4a523af04d 2024-11-14T21:33:46,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-14T21:33:46,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:46,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:46,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:46,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:46,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:46,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/9212aab04e7f475894c299a12fa72a2a is 50, key is test_row_0/C:col10/1731620024410/Put/seqid=0 2024-11-14T21:33:46,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742181_1357 (size=12001) 2024-11-14T21:33:46,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620086480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:46,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620086480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:46,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:46,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:46,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:46,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620086491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:46,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:46,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:46,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:46,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-14T21:33:46,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:46,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:46,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:46,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:46,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:46,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620086541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:46,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620086543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:46,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:46,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:46,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-14T21:33:46,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:46,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:46,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:46,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:46,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:46,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-14T21:33:46,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:46,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:46,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:46,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:46,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:46,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/9212aab04e7f475894c299a12fa72a2a 2024-11-14T21:33:46,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/db941d86545e4188893d73292fbae917 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/db941d86545e4188893d73292fbae917 2024-11-14T21:33:46,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/db941d86545e4188893d73292fbae917, entries=150, sequenceid=85, filesize=30.2 K 2024-11-14T21:33:46,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/d9aefa8d9bef408493f1cf4a523af04d as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/d9aefa8d9bef408493f1cf4a523af04d 2024-11-14T21:33:46,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/d9aefa8d9bef408493f1cf4a523af04d, entries=150, sequenceid=85, filesize=11.7 K 2024-11-14T21:33:46,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/9212aab04e7f475894c299a12fa72a2a as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/9212aab04e7f475894c299a12fa72a2a 2024-11-14T21:33:46,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/9212aab04e7f475894c299a12fa72a2a, entries=150, sequenceid=85, filesize=11.7 K 2024-11-14T21:33:46,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for c47d7036b7faae0ba1cfffe0348781b1 in 1281ms, sequenceid=85, compaction requested=true 2024-11-14T21:33:46,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:46,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:33:46,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:46,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:33:46,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:33:46,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:33:46,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T21:33:46,949 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:46,949 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:46,959 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:46,959 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/A is initiating minor compaction (all files) 2024-11-14T21:33:46,959 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/A in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:46,959 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/dcb4d04957aa45dd851d316c885db620, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/c7f88ae6aa05447b9ec27eeaee3e8193, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/db941d86545e4188893d73292fbae917] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=99.2 K 2024-11-14T21:33:46,959 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:46,959 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. files: [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/dcb4d04957aa45dd851d316c885db620, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/c7f88ae6aa05447b9ec27eeaee3e8193, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/db941d86545e4188893d73292fbae917] 2024-11-14T21:33:46,962 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:46,962 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/C is initiating minor compaction (all files) 2024-11-14T21:33:46,962 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/C in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:46,962 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/cf78ff02093e49a383d8505ba7a2b583, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/2fdce2cb2fea4d46803ad70ddc682157, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/9212aab04e7f475894c299a12fa72a2a] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=35.3 K 2024-11-14T21:33:46,966 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting dcb4d04957aa45dd851d316c885db620, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731620023313 2024-11-14T21:33:46,969 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf78ff02093e49a383d8505ba7a2b583, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731620023313 2024-11-14T21:33:46,969 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting c7f88ae6aa05447b9ec27eeaee3e8193, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731620023711 2024-11-14T21:33:46,978 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fdce2cb2fea4d46803ad70ddc682157, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731620023711 2024-11-14T21:33:46,981 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9212aab04e7f475894c299a12fa72a2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731620024410 2024-11-14T21:33:46,982 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting db941d86545e4188893d73292fbae917, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731620024410 2024-11-14T21:33:47,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731620025765, max timestamp=1731620027003Num uniques 859; 2024-11-14T21:33:47,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:33:47,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731620025765, max timestamp=1731620027003Num uniques 859; 2024-11-14T21:33:47,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:33:47,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-14T21:33:47,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:47,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731620025765, max timestamp=1731620027003Num uniques 859; 2024-11-14T21:33:47,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:33:47,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:47,015 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2902): Flushing c47d7036b7faae0ba1cfffe0348781b1 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-11-14T21:33:47,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=A 2024-11-14T21:33:47,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:47,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=B 2024-11-14T21:33:47,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:47,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=C 2024-11-14T21:33:47,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:47,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:47,029 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#C#compaction#308 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:47,029 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/e5d3ebf2f9754d10bd86b39dfbe41bae is 50, key is test_row_0/C:col10/1731620024410/Put/seqid=0 2024-11-14T21:33:47,031 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:47,047 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114ef505238606c448a8c4a41870d5c720c_c47d7036b7faae0ba1cfffe0348781b1 store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:47,049 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114ef505238606c448a8c4a41870d5c720c_c47d7036b7faae0ba1cfffe0348781b1, store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:47,049 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ef505238606c448a8c4a41870d5c720c_c47d7036b7faae0ba1cfffe0348781b1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:47,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742182_1358 (size=12207) 2024-11-14T21:33:47,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114db9e6b157780495894213978628df539_c47d7036b7faae0ba1cfffe0348781b1 is 50, key is test_row_0/A:col10/1731620025788/Put/seqid=0 2024-11-14T21:33:47,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742183_1359 (size=4469) 2024-11-14T21:33:47,073 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#A#compaction#309 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:47,074 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/e4f7118d1186475aa4266d77af35c471 is 175, key is test_row_0/A:col10/1731620024410/Put/seqid=0 2024-11-14T21:33:47,075 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/e5d3ebf2f9754d10bd86b39dfbe41bae as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/e5d3ebf2f9754d10bd86b39dfbe41bae 2024-11-14T21:33:47,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742184_1360 (size=12154) 2024-11-14T21:33:47,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742185_1361 (size=31161) 2024-11-14T21:33:47,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731620027033, max timestamp=1731620027105Num uniques 859; 2024-11-14T21:33:47,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731620027033, max timestamp=1731620027105Num uniques 859; 2024-11-14T21:33:47,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T21:33:47,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T21:33:47,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731620027033, max timestamp=1731620027105Num uniques 859; 2024-11-14T21:33:47,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T21:33:47,115 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/C of c47d7036b7faae0ba1cfffe0348781b1 into e5d3ebf2f9754d10bd86b39dfbe41bae(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:47,115 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:47,115 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/C, priority=13, startTime=1731620026947; duration=0sec 2024-11-14T21:33:47,115 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:47,115 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:C 2024-11-14T21:33:47,115 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:47,127 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:47,127 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/B is initiating minor compaction (all files) 2024-11-14T21:33:47,127 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/B in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:47,127 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/00ce6fe3e6c0436185946eea3a67aed8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/133c3a15ff7c482093ec134fd4e19b3e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/d9aefa8d9bef408493f1cf4a523af04d] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=35.3 K 2024-11-14T21:33:47,127 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00ce6fe3e6c0436185946eea3a67aed8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731620023313 2024-11-14T21:33:47,128 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 133c3a15ff7c482093ec134fd4e19b3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731620023711 2024-11-14T21:33:47,128 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9aefa8d9bef408493f1cf4a523af04d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731620024410 2024-11-14T21:33:47,149 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#B#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:47,150 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/a62b8dd604ef47f4ba9a7bce7052eccd is 50, key is test_row_0/B:col10/1731620024410/Put/seqid=0 2024-11-14T21:33:47,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742186_1362 (size=12207) 2024-11-14T21:33:47,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:47,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:47,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620087183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:47,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620087184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:47,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:47,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620087184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:47,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:47,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:47,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:47,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:47,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:47,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:47,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:47,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:47,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:47,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:47,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620087201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:47,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731620027176, max timestamp=1731620027200Num uniques 1168; 2024-11-14T21:33:47,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=6870, suffix heap size=149440,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=150 2024-11-14T21:33:47,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731620027176, max timestamp=1731620027200Num uniques 1168; 2024-11-14T21:33:47,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=6870, suffix heap size=149440,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=150 2024-11-14T21:33:47,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731620027176, max timestamp=1731620027200Num uniques 1168; 2024-11-14T21:33:47,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=6870, suffix heap size=149440,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=150 2024-11-14T21:33:47,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731620027348, max timestamp=1731620027373Num uniques 1009; 2024-11-14T21:33:47,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=12682, suffix heap size=132904,new segment heap size=29048  suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=277 2024-11-14T21:33:47,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731620027348, max timestamp=1731620027373Num uniques 1009; 2024-11-14T21:33:47,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=12682, suffix heap size=132904,new segment heap size=29048  suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=277 2024-11-14T21:33:47,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731620027348, max timestamp=1731620027373Num uniques 1009; 2024-11-14T21:33:47,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=12682, suffix heap size=132904,new segment heap size=29048  suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=277 2024-11-14T21:33:47,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:47,506 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114db9e6b157780495894213978628df539_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114db9e6b157780495894213978628df539_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:47,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/af01915940134012bf0ed52f9c00ceb8, store: [table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:47,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/af01915940134012bf0ed52f9c00ceb8 is 175, key is test_row_0/A:col10/1731620025788/Put/seqid=0 2024-11-14T21:33:47,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742187_1363 (size=30955) 2024-11-14T21:33:47,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731620027479, max timestamp=1731620027531Num uniques 1136; 2024-11-14T21:33:47,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984  suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-11-14T21:33:47,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731620027479, max timestamp=1731620027531Num uniques 1136; 2024-11-14T21:33:47,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984  suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-11-14T21:33:47,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731620027479, max timestamp=1731620027531Num uniques 1136; 2024-11-14T21:33:47,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984  suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-11-14T21:33:47,546 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=106, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/af01915940134012bf0ed52f9c00ceb8 2024-11-14T21:33:47,546 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/e4f7118d1186475aa4266d77af35c471 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/e4f7118d1186475aa4266d77af35c471 2024-11-14T21:33:47,557 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/A of c47d7036b7faae0ba1cfffe0348781b1 into e4f7118d1186475aa4266d77af35c471(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:47,557 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:47,557 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/A, priority=13, startTime=1731620026947; duration=0sec 2024-11-14T21:33:47,557 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:47,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/1b08e0ab86754302a99cc8826d8c166d is 50, key is test_row_0/B:col10/1731620025788/Put/seqid=0 2024-11-14T21:33:47,557 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:A 2024-11-14T21:33:47,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742188_1364 (size=12001) 2024-11-14T21:33:47,575 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/1b08e0ab86754302a99cc8826d8c166d 2024-11-14T21:33:47,589 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/a62b8dd604ef47f4ba9a7bce7052eccd as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/a62b8dd604ef47f4ba9a7bce7052eccd 2024-11-14T21:33:47,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/8d7277662164477cbadc589f053d8b0b is 50, key is test_row_0/C:col10/1731620025788/Put/seqid=0 2024-11-14T21:33:47,643 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/B of c47d7036b7faae0ba1cfffe0348781b1 into a62b8dd604ef47f4ba9a7bce7052eccd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:47,643 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:47,643 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/B, priority=13, startTime=1731620026947; duration=0sec 2024-11-14T21:33:47,643 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:47,643 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:B 2024-11-14T21:33:47,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742189_1365 (size=12001) 2024-11-14T21:33:47,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731620027587, max timestamp=1731620027675Num uniques 1145; 2024-11-14T21:33:47,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731620027587, max timestamp=1731620027675Num uniques 1145; 2024-11-14T21:33:47,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520  suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-14T21:33:47,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520  suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-14T21:33:47,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8920, totalHeapSize=20520, min timestamp=1731620027587, max timestamp=1731620027681Num uniques 1145; 2024-11-14T21:33:47,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52430, new segment data size=8920, suffix heap size=147048,new segment heap size=20520  suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-14T21:33:47,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731620027738, max timestamp=1731620027773Num uniques 1054; 2024-11-14T21:33:47,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456  suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-14T21:33:47,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731620027738, max timestamp=1731620027773Num uniques 1054; 2024-11-14T21:33:47,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731620027738, max timestamp=1731620027773Num uniques 1054; 2024-11-14T21:33:47,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456  suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-14T21:33:47,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456  suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-14T21:33:47,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731620027830, max timestamp=1731620027858Num uniques 1063; 2024-11-14T21:33:47,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392  suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-14T21:33:47,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731620027830, max timestamp=1731620027858Num uniques 1063; 2024-11-14T21:33:47,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392  suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-14T21:33:47,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731620027830, max timestamp=1731620027858Num uniques 1063; 2024-11-14T21:33:47,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392  suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-14T21:33:47,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731620027920, max timestamp=1731620027953Num uniques 1072; 2024-11-14T21:33:47,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731620027920, max timestamp=1731620027953Num uniques 1072; 2024-11-14T21:33:47,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128  suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-14T21:33:47,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128  suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-14T21:33:47,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731620027920, max timestamp=1731620027953Num uniques 1072; 2024-11-14T21:33:47,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128  suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-14T21:33:48,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731620028045, max timestamp=1731620028075Num uniques 1031; 2024-11-14T21:33:48,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264  suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-14T21:33:48,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731620028045, max timestamp=1731620028075Num uniques 1031; 2024-11-14T21:33:48,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264  suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-14T21:33:48,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731620028045, max timestamp=1731620028075Num uniques 1031; 2024-11-14T21:33:48,087 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/8d7277662164477cbadc589f053d8b0b 2024-11-14T21:33:48,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264  suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-14T21:33:48,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/af01915940134012bf0ed52f9c00ceb8 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/af01915940134012bf0ed52f9c00ceb8 2024-11-14T21:33:48,119 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/af01915940134012bf0ed52f9c00ceb8, entries=150, sequenceid=106, filesize=30.2 K 2024-11-14T21:33:48,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/1b08e0ab86754302a99cc8826d8c166d as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/1b08e0ab86754302a99cc8826d8c166d 2024-11-14T21:33:48,127 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/1b08e0ab86754302a99cc8826d8c166d, entries=150, sequenceid=106, filesize=11.7 K 2024-11-14T21:33:48,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/8d7277662164477cbadc589f053d8b0b as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/8d7277662164477cbadc589f053d8b0b 2024-11-14T21:33:48,135 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/8d7277662164477cbadc589f053d8b0b, entries=150, sequenceid=106, filesize=11.7 K 2024-11-14T21:33:48,138 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=114.05 KB/116790 for c47d7036b7faae0ba1cfffe0348781b1 in 1124ms, sequenceid=106, compaction requested=false 2024-11-14T21:33:48,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2603): Flush status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:48,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:48,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-11-14T21:33:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=86 2024-11-14T21:33:48,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c47d7036b7faae0ba1cfffe0348781b1 3/3 column families, dataSize=120.76 KB heapSize=338.39 KB 2024-11-14T21:33:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:48,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=A 2024-11-14T21:33:48,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:48,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=B 2024-11-14T21:33:48,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:48,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=C 2024-11-14T21:33:48,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:48,144 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-14T21:33:48,145 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.0150 sec 2024-11-14T21:33:48,146 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees in 4.0220 sec 2024-11-14T21:33:48,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e93f6be5ef5a4b0da5bc78c6269f464d_c47d7036b7faae0ba1cfffe0348781b1 is 50, key is test_row_0/A:col10/1731620028134/Put/seqid=0 2024-11-14T21:33:48,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620088153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:48,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620088156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:48,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742190_1366 (size=20074) 2024-11-14T21:33:48,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620088157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:48,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:48,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620088159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:48,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:48,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:48,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620088167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:48,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:48,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620088270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:48,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620088270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:48,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620088280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:48,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:48,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620088280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:48,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:48,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620088280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T21:33:48,311 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:33:48,311 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:33:48,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-11-14T21:33:48,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T21:33:48,315 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:33:48,316 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:33:48,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:33:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T21:33:48,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T21:33:48,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:48,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:48,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:48,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:48,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:48,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620088491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:48,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620088491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:48,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620088500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:48,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620088500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:48,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:48,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620088505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:48,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:48,572 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:48,576 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e93f6be5ef5a4b0da5bc78c6269f464d_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e93f6be5ef5a4b0da5bc78c6269f464d_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:48,577 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/9faa843d05614d9da4e5a92a8794def1, store: [table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:48,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/9faa843d05614d9da4e5a92a8794def1 is 175, key is test_row_0/A:col10/1731620028134/Put/seqid=0 2024-11-14T21:33:48,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742191_1367 (size=57333) 2024-11-14T21:33:48,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T21:33:48,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:48,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:48,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:48,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:48,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:48,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T21:33:48,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T21:33:48,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:48,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:48,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:48,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:48,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:48,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620088813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:48,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:48,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620088813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:48,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:48,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:48,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:48,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620088820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:48,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:48,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620088820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:48,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:48,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:48,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:48,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620088823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:48,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:48,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:48,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T21:33:48,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:48,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:48,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:48,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:48,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:48,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T21:33:48,984 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/9faa843d05614d9da4e5a92a8794def1 2024-11-14T21:33:48,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/b8f22d5ea4cb4fe89bcff1c7668cd013 is 50, key is test_row_0/B:col10/1731620028134/Put/seqid=0 2024-11-14T21:33:49,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742192_1368 (size=12301) 2024-11-14T21:33:49,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T21:33:49,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:49,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:49,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:49,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:49,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:49,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:49,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T21:33:49,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:49,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:49,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:49,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:49,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:49,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:49,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:49,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620089331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:49,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:49,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:49,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:49,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620089331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:49,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:49,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:49,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:49,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:49,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:49,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620089343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:49,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:49,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:49,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:49,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:49,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620089343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:49,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:49,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:49,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:49,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:49,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620089343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:49,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:49,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:49,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:49,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T21:33:49,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:49,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:49,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:49,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:49,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:49,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:49,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/b8f22d5ea4cb4fe89bcff1c7668cd013 2024-11-14T21:33:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T21:33:49,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/c53453a56a1a4f4cbeaee5e8040bacd2 is 50, key is test_row_0/C:col10/1731620028134/Put/seqid=0 2024-11-14T21:33:49,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742193_1369 (size=12301) 2024-11-14T21:33:49,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/c53453a56a1a4f4cbeaee5e8040bacd2 2024-11-14T21:33:49,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/9faa843d05614d9da4e5a92a8794def1 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/9faa843d05614d9da4e5a92a8794def1 2024-11-14T21:33:49,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/9faa843d05614d9da4e5a92a8794def1, entries=300, sequenceid=281, filesize=56.0 K 2024-11-14T21:33:49,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T21:33:49,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:49,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:49,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:49,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:49,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:49,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/b8f22d5ea4cb4fe89bcff1c7668cd013 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/b8f22d5ea4cb4fe89bcff1c7668cd013 2024-11-14T21:33:49,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/b8f22d5ea4cb4fe89bcff1c7668cd013, entries=150, sequenceid=281, filesize=12.0 K 2024-11-14T21:33:49,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/c53453a56a1a4f4cbeaee5e8040bacd2 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/c53453a56a1a4f4cbeaee5e8040bacd2 2024-11-14T21:33:49,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/c53453a56a1a4f4cbeaee5e8040bacd2, entries=150, sequenceid=281, filesize=12.0 K 2024-11-14T21:33:49,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~358.27 KB/366864, currentSize=53.67 KB/54960 for c47d7036b7faae0ba1cfffe0348781b1 in 1470ms, sequenceid=281, compaction requested=true 2024-11-14T21:33:49,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:49,610 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:49,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:33:49,611 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:49,611 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/A is initiating minor compaction (all files) 2024-11-14T21:33:49,611 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/A in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:49,611 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/e4f7118d1186475aa4266d77af35c471, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/af01915940134012bf0ed52f9c00ceb8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/9faa843d05614d9da4e5a92a8794def1] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=116.6 K 2024-11-14T21:33:49,611 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:49,611 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. files: [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/e4f7118d1186475aa4266d77af35c471, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/af01915940134012bf0ed52f9c00ceb8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/9faa843d05614d9da4e5a92a8794def1] 2024-11-14T21:33:49,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:49,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:33:49,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:49,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:33:49,611 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:49,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:49,612 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4f7118d1186475aa4266d77af35c471, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731620024410 2024-11-14T21:33:49,615 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting af01915940134012bf0ed52f9c00ceb8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731620025788 2024-11-14T21:33:49,622 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9faa843d05614d9da4e5a92a8794def1, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731620028114 2024-11-14T21:33:49,622 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:49,622 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/B is initiating minor compaction (all files) 2024-11-14T21:33:49,622 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/B in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:49,622 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/a62b8dd604ef47f4ba9a7bce7052eccd, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/1b08e0ab86754302a99cc8826d8c166d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/b8f22d5ea4cb4fe89bcff1c7668cd013] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=35.7 K 2024-11-14T21:33:49,624 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting a62b8dd604ef47f4ba9a7bce7052eccd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731620024410 2024-11-14T21:33:49,624 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b08e0ab86754302a99cc8826d8c166d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731620025788 2024-11-14T21:33:49,631 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting b8f22d5ea4cb4fe89bcff1c7668cd013, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731620028134 2024-11-14T21:33:49,640 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:49,642 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#B#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:49,643 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/70d816227f034568a0eb2d0a69281ddb is 50, key is test_row_0/B:col10/1731620028134/Put/seqid=0 2024-11-14T21:33:49,657 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411142dfe5282d6b1452e82d1ad202fb7d0ee_c47d7036b7faae0ba1cfffe0348781b1 store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:49,659 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411142dfe5282d6b1452e82d1ad202fb7d0ee_c47d7036b7faae0ba1cfffe0348781b1, store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:49,659 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142dfe5282d6b1452e82d1ad202fb7d0ee_c47d7036b7faae0ba1cfffe0348781b1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:49,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742194_1370 (size=12609) 2024-11-14T21:33:49,680 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/70d816227f034568a0eb2d0a69281ddb as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/70d816227f034568a0eb2d0a69281ddb 2024-11-14T21:33:49,686 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/B of c47d7036b7faae0ba1cfffe0348781b1 into 70d816227f034568a0eb2d0a69281ddb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:49,686 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:49,686 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/B, priority=13, startTime=1731620029611; duration=0sec 2024-11-14T21:33:49,686 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:49,686 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:B 2024-11-14T21:33:49,686 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:49,687 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:49,687 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/C is initiating minor compaction (all files) 2024-11-14T21:33:49,687 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/C in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:49,688 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/e5d3ebf2f9754d10bd86b39dfbe41bae, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/8d7277662164477cbadc589f053d8b0b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/c53453a56a1a4f4cbeaee5e8040bacd2] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=35.7 K 2024-11-14T21:33:49,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742195_1371 (size=4469) 2024-11-14T21:33:49,689 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting e5d3ebf2f9754d10bd86b39dfbe41bae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731620024410 2024-11-14T21:33:49,689 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d7277662164477cbadc589f053d8b0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731620025788 2024-11-14T21:33:49,689 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting c53453a56a1a4f4cbeaee5e8040bacd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731620028134 2024-11-14T21:33:49,712 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#C#compaction#319 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:49,713 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/60dbe313e3bf429ca21251e16098ed91 is 50, key is test_row_0/C:col10/1731620028134/Put/seqid=0 2024-11-14T21:33:49,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742196_1372 (size=12609) 2024-11-14T21:33:49,735 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/60dbe313e3bf429ca21251e16098ed91 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/60dbe313e3bf429ca21251e16098ed91 2024-11-14T21:33:49,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T21:33:49,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:49,742 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing c47d7036b7faae0ba1cfffe0348781b1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T21:33:49,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=A 2024-11-14T21:33:49,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:49,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=B 2024-11-14T21:33:49,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:49,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=C 2024-11-14T21:33:49,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:49,747 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/C of c47d7036b7faae0ba1cfffe0348781b1 into 60dbe313e3bf429ca21251e16098ed91(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:49,747 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:49,747 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/C, priority=13, startTime=1731620029611; duration=0sec 2024-11-14T21:33:49,747 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:49,747 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:C 2024-11-14T21:33:49,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111486a8fe1e1846499a93cba06563fc6c40_c47d7036b7faae0ba1cfffe0348781b1 is 50, key is test_row_0/A:col10/1731620028157/Put/seqid=0 2024-11-14T21:33:49,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742197_1373 (size=12454) 2024-11-14T21:33:50,095 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#A#compaction#317 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:50,096 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/e00f83f2eec24d258ef81dd69b00fc67 is 175, key is test_row_0/A:col10/1731620028134/Put/seqid=0 2024-11-14T21:33:50,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742198_1374 (size=31563) 2024-11-14T21:33:50,151 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/e00f83f2eec24d258ef81dd69b00fc67 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/e00f83f2eec24d258ef81dd69b00fc67 2024-11-14T21:33:50,165 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/A of c47d7036b7faae0ba1cfffe0348781b1 into e00f83f2eec24d258ef81dd69b00fc67(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:50,165 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:50,165 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/A, priority=13, startTime=1731620029610; duration=0sec 2024-11-14T21:33:50,165 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:50,165 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:A 2024-11-14T21:33:50,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:50,227 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111486a8fe1e1846499a93cba06563fc6c40_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111486a8fe1e1846499a93cba06563fc6c40_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:50,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/ca40ba5f86dd4c048d26b0f99fcf2d82, store: [table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:50,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/ca40ba5f86dd4c048d26b0f99fcf2d82 is 175, key is test_row_0/A:col10/1731620028157/Put/seqid=0 2024-11-14T21:33:50,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742199_1375 (size=31255) 2024-11-14T21:33:50,281 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/ca40ba5f86dd4c048d26b0f99fcf2d82 2024-11-14T21:33:50,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/771bae3374ea4d499c97eec94d91ad4b is 50, key is test_row_0/B:col10/1731620028157/Put/seqid=0 2024-11-14T21:33:50,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742200_1376 (size=12301) 2024-11-14T21:33:50,349 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/771bae3374ea4d499c97eec94d91ad4b 2024-11-14T21:33:50,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:50,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:50,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/dafbc51b457249afa2976596a8584201 is 50, key is test_row_0/C:col10/1731620028157/Put/seqid=0 2024-11-14T21:33:50,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742201_1377 (size=12301) 2024-11-14T21:33:50,410 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/dafbc51b457249afa2976596a8584201 2024-11-14T21:33:50,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/ca40ba5f86dd4c048d26b0f99fcf2d82 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/ca40ba5f86dd4c048d26b0f99fcf2d82 2024-11-14T21:33:50,420 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/ca40ba5f86dd4c048d26b0f99fcf2d82, entries=150, sequenceid=293, filesize=30.5 K 2024-11-14T21:33:50,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/771bae3374ea4d499c97eec94d91ad4b as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/771bae3374ea4d499c97eec94d91ad4b 2024-11-14T21:33:50,451 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/771bae3374ea4d499c97eec94d91ad4b, entries=150, sequenceid=293, filesize=12.0 K 2024-11-14T21:33:50,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/dafbc51b457249afa2976596a8584201 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/dafbc51b457249afa2976596a8584201 2024-11-14T21:33:50,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T21:33:50,471 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/dafbc51b457249afa2976596a8584201, entries=150, sequenceid=293, filesize=12.0 K 2024-11-14T21:33:50,480 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=114.05 KB/116790 for c47d7036b7faae0ba1cfffe0348781b1 in 738ms, sequenceid=293, compaction requested=false 2024-11-14T21:33:50,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:50,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:50,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-14T21:33:50,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731620030421, max timestamp=1731620030469Num uniques 859; 2024-11-14T21:33:50,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:33:50,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731620030421, max timestamp=1731620030469Num uniques 859; 2024-11-14T21:33:50,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:33:50,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:50,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c47d7036b7faae0ba1cfffe0348781b1 3/3 column families, dataSize=64.85 KB heapSize=184.09 KB 2024-11-14T21:33:50,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=A 2024-11-14T21:33:50,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:50,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=B 2024-11-14T21:33:50,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731620030421, max timestamp=1731620030469Num uniques 859; 2024-11-14T21:33:50,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:33:50,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:50,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=C 2024-11-14T21:33:50,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-11-14T21:33:50,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bafbfaf515a641c386a697e275cec94e_c47d7036b7faae0ba1cfffe0348781b1 is 50, key is test_row_0/A:col10/1731620030468/Put/seqid=0 2024-11-14T21:33:50,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-14T21:33:50,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1680 sec 2024-11-14T21:33:50,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 2.1950 sec 2024-11-14T21:33:50,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742202_1378 (size=17534) 2024-11-14T21:33:50,563 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:50,569 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bafbfaf515a641c386a697e275cec94e_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bafbfaf515a641c386a697e275cec94e_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:50,570 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/2bdd0a8d5746431887a773856ad590d1, store: [table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:50,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/2bdd0a8d5746431887a773856ad590d1 is 175, key is test_row_0/A:col10/1731620030468/Put/seqid=0 2024-11-14T21:33:50,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742203_1379 (size=48639) 2024-11-14T21:33:50,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731620030557, max timestamp=1731620030605Num uniques 859; 2024-11-14T21:33:50,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:33:50,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731620030557, max timestamp=1731620030605Num uniques 859; 2024-11-14T21:33:50,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731620030557, max timestamp=1731620030605Num uniques 859; 2024-11-14T21:33:50,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:33:50,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:33:50,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731620030630, max timestamp=1731620030707Num uniques 1118; 2024-11-14T21:33:50,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312  suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-11-14T21:33:50,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731620030630, max timestamp=1731620030707Num uniques 1118; 2024-11-14T21:33:50,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312  suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-11-14T21:33:50,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731620030630, max timestamp=1731620030707Num uniques 1118; 2024-11-14T21:33:50,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312  suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-11-14T21:33:50,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620090790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:50,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:50,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620090797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:50,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620090798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:50,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620090809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:50,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:50,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:50,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:50,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:50,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:50,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:50,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620090809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:50,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:50,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:50,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:50,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620090921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:50,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:50,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:50,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:50,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:50,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620090921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:50,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:50,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:50,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:50,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620090921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:50,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:50,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620090921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:50,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620090922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:51,015 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/2bdd0a8d5746431887a773856ad590d1 2024-11-14T21:33:51,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/d3d046faa0724dd29de9102a76cf85d8 is 50, key is test_row_0/B:col10/1731620030468/Put/seqid=0 2024-11-14T21:33:51,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742204_1380 (size=12301) 2024-11-14T21:33:51,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620091143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:51,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620091149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:51,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:51,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620091150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:51,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620091151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:51,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620091151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:51,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:51,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:51,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:51,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/d3d046faa0724dd29de9102a76cf85d8 2024-11-14T21:33:51,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620091483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:51,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:51,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:51,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:51,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620091483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:51,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620091483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:51,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620091484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:51,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620091486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:51,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/6d1044c81b4f430b933d5578be5a53d4 is 50, key is test_row_0/C:col10/1731620030468/Put/seqid=0 2024-11-14T21:33:51,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742205_1381 (size=12301) 2024-11-14T21:33:51,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/6d1044c81b4f430b933d5578be5a53d4 2024-11-14T21:33:51,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/2bdd0a8d5746431887a773856ad590d1 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/2bdd0a8d5746431887a773856ad590d1 2024-11-14T21:33:51,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/2bdd0a8d5746431887a773856ad590d1, entries=250, sequenceid=317, filesize=47.5 K 2024-11-14T21:33:51,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/d3d046faa0724dd29de9102a76cf85d8 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/d3d046faa0724dd29de9102a76cf85d8 2024-11-14T21:33:51,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/d3d046faa0724dd29de9102a76cf85d8, entries=150, sequenceid=317, filesize=12.0 K 2024-11-14T21:33:51,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/6d1044c81b4f430b933d5578be5a53d4 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/6d1044c81b4f430b933d5578be5a53d4 2024-11-14T21:33:51,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/6d1044c81b4f430b933d5578be5a53d4, entries=150, sequenceid=317, filesize=12.0 K 2024-11-14T21:33:51,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=154.31 KB/158010 for c47d7036b7faae0ba1cfffe0348781b1 in 1129ms, sequenceid=317, compaction requested=true 2024-11-14T21:33:51,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:51,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:33:51,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:51,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:33:51,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:51,612 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:51,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:33:51,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:33:51,612 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:51,613 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111457 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:51,613 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:51,613 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/A is initiating minor compaction (all files) 2024-11-14T21:33:51,613 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/B is initiating minor compaction (all files) 2024-11-14T21:33:51,613 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/A in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:51,613 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/B in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:51,613 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/e00f83f2eec24d258ef81dd69b00fc67, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/ca40ba5f86dd4c048d26b0f99fcf2d82, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/2bdd0a8d5746431887a773856ad590d1] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=108.8 K 2024-11-14T21:33:51,613 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/70d816227f034568a0eb2d0a69281ddb, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/771bae3374ea4d499c97eec94d91ad4b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/d3d046faa0724dd29de9102a76cf85d8] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=36.3 K 2024-11-14T21:33:51,613 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:51,613 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. files: [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/e00f83f2eec24d258ef81dd69b00fc67, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/ca40ba5f86dd4c048d26b0f99fcf2d82, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/2bdd0a8d5746431887a773856ad590d1] 2024-11-14T21:33:51,614 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70d816227f034568a0eb2d0a69281ddb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731620028134 2024-11-14T21:33:51,614 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting e00f83f2eec24d258ef81dd69b00fc67, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731620028134 2024-11-14T21:33:51,614 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting ca40ba5f86dd4c048d26b0f99fcf2d82, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731620028142 2024-11-14T21:33:51,615 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bdd0a8d5746431887a773856ad590d1, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731620030421 2024-11-14T21:33:51,616 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 771bae3374ea4d499c97eec94d91ad4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731620028142 2024-11-14T21:33:51,617 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3d046faa0724dd29de9102a76cf85d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731620030468 2024-11-14T21:33:51,627 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:51,630 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#B#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:51,631 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/c6b117c1b4d048129c61c33d2c0e4404 is 50, key is test_row_0/B:col10/1731620030468/Put/seqid=0 2024-11-14T21:33:51,656 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114dfa041f5b010423d8bc87ea7fb55aaaf_c47d7036b7faae0ba1cfffe0348781b1 store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:51,658 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114dfa041f5b010423d8bc87ea7fb55aaaf_c47d7036b7faae0ba1cfffe0348781b1, store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:51,658 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114dfa041f5b010423d8bc87ea7fb55aaaf_c47d7036b7faae0ba1cfffe0348781b1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:51,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742206_1382 (size=12711) 2024-11-14T21:33:51,699 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/c6b117c1b4d048129c61c33d2c0e4404 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/c6b117c1b4d048129c61c33d2c0e4404 2024-11-14T21:33:51,721 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/B of c47d7036b7faae0ba1cfffe0348781b1 into c6b117c1b4d048129c61c33d2c0e4404(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:51,721 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:51,721 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/B, priority=13, startTime=1731620031612; duration=0sec 2024-11-14T21:33:51,721 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:51,721 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:B 2024-11-14T21:33:51,721 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:51,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742207_1383 (size=4469) 2024-11-14T21:33:51,733 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:51,733 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/C is initiating minor compaction (all files) 2024-11-14T21:33:51,733 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/C in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:51,733 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/60dbe313e3bf429ca21251e16098ed91, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/dafbc51b457249afa2976596a8584201, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/6d1044c81b4f430b933d5578be5a53d4] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=36.3 K 2024-11-14T21:33:51,739 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60dbe313e3bf429ca21251e16098ed91, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731620028134 2024-11-14T21:33:51,740 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting dafbc51b457249afa2976596a8584201, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731620028142 2024-11-14T21:33:51,743 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d1044c81b4f430b933d5578be5a53d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731620030468 2024-11-14T21:33:51,743 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#A#compaction#326 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:51,744 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/21dd5fa1c92d46b8a4516e0cbade204c is 175, key is test_row_0/A:col10/1731620030468/Put/seqid=0 2024-11-14T21:33:51,780 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#C#compaction#328 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:51,781 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/5ef3f887e4bd445383413faf3f591b50 is 50, key is test_row_0/C:col10/1731620030468/Put/seqid=0 2024-11-14T21:33:51,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742208_1384 (size=31665) 2024-11-14T21:33:51,818 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/21dd5fa1c92d46b8a4516e0cbade204c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/21dd5fa1c92d46b8a4516e0cbade204c 2024-11-14T21:33:51,828 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/A of c47d7036b7faae0ba1cfffe0348781b1 into 21dd5fa1c92d46b8a4516e0cbade204c(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:51,828 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:51,828 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/A, priority=13, startTime=1731620031612; duration=0sec 2024-11-14T21:33:51,828 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:51,828 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:A 2024-11-14T21:33:51,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742209_1385 (size=12711) 2024-11-14T21:33:52,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731620030788, max timestamp=1731620032015Num uniques 1177; 2024-11-14T21:33:52,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=8102, suffix heap size=150376,new segment heap size=18648  suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=177 2024-11-14T21:33:52,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:52,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c47d7036b7faae0ba1cfffe0348781b1 3/3 column families, dataSize=116.29 KB heapSize=321.83 KB 2024-11-14T21:33:52,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=A 2024-11-14T21:33:52,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T21:33:52,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=B 2024-11-14T21:33:52,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:52,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=C 2024-11-14T21:33:52,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T21:33:52,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fc5f3ec123794c5c891f1a92f8e74609_c47d7036b7faae0ba1cfffe0348781b1 is 50, key is test_row_0/A:col10/1731620032015/Put/seqid=0 2024-11-14T21:33:52,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620092046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:52,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620092049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:52,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620092051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:52,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742210_1386 (size=14994) 2024-11-14T21:33:52,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:52,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620092065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:52,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620092079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:52,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:52,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620092191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:52,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:52,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620092191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:52,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620092191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:52,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620092202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:52,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620092203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:52,246 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/5ef3f887e4bd445383413faf3f591b50 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/5ef3f887e4bd445383413faf3f591b50 2024-11-14T21:33:52,251 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/C of c47d7036b7faae0ba1cfffe0348781b1 into 5ef3f887e4bd445383413faf3f591b50(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:52,251 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:52,251 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/C, priority=13, startTime=1731620031612; duration=0sec 2024-11-14T21:33:52,251 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:52,251 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:C 2024-11-14T21:33:52,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620092424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:52,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620092424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:52,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620092424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:52,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620092424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:52,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:52,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:52,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T21:33:52,484 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:33:52,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:33:52,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620092430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-14T21:33:52,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:52,491 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:33:52,494 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:52,495 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:33:52,496 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:33:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T21:33:52,510 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fc5f3ec123794c5c891f1a92f8e74609_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fc5f3ec123794c5c891f1a92f8e74609_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:52,512 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/31d79eb7e0b941d280d3524dfceed912, store: [table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:52,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/31d79eb7e0b941d280d3524dfceed912 is 175, key is test_row_0/A:col10/1731620032015/Put/seqid=0 2024-11-14T21:33:52,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742211_1387 (size=39949) 2024-11-14T21:33:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T21:33:52,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T21:33:52,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:52,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:52,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:52,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:52,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:52,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620092751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:52,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620092751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:52,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:52,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620092791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:52,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T21:33:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:52,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:52,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:52,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:52,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620092791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:52,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:52,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T21:33:52,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620092800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:52,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:52,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:52,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T21:33:52,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:52,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:52,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:52,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:52,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:52,961 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=374, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/31d79eb7e0b941d280d3524dfceed912 2024-11-14T21:33:52,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/c3dba67a85b54023931d1542a07267d6 is 50, key is test_row_0/B:col10/1731620032015/Put/seqid=0 2024-11-14T21:33:53,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742212_1388 (size=12301) 2024-11-14T21:33:53,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T21:33:53,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:53,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:53,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:53,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:53,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:53,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T21:33:53,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T21:33:53,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:53,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:53,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:53,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:53,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:53,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:53,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:53,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620093271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:53,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:53,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:53,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:53,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:53,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620093271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:53,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:53,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:53,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:53,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620093311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:53,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620093311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:53,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:53,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620093331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:53,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:53,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:53,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:53,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T21:33:53,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:53,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:53,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:53,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:53,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:53,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:53,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/c3dba67a85b54023931d1542a07267d6 2024-11-14T21:33:53,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/ba7b059c0e5845159c6d029b95cf621f is 50, key is test_row_0/C:col10/1731620032015/Put/seqid=0 2024-11-14T21:33:53,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742213_1389 (size=12301) 2024-11-14T21:33:53,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/ba7b059c0e5845159c6d029b95cf621f 2024-11-14T21:33:53,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/31d79eb7e0b941d280d3524dfceed912 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/31d79eb7e0b941d280d3524dfceed912 2024-11-14T21:33:53,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/31d79eb7e0b941d280d3524dfceed912, entries=200, sequenceid=374, filesize=39.0 K 2024-11-14T21:33:53,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/c3dba67a85b54023931d1542a07267d6 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/c3dba67a85b54023931d1542a07267d6 2024-11-14T21:33:53,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/c3dba67a85b54023931d1542a07267d6, entries=150, sequenceid=374, filesize=12.0 K 2024-11-14T21:33:53,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/ba7b059c0e5845159c6d029b95cf621f as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/ba7b059c0e5845159c6d029b95cf621f 2024-11-14T21:33:53,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T21:33:53,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:53,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:53,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:53,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:53,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:53,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:53,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/ba7b059c0e5845159c6d029b95cf621f, entries=150, sequenceid=374, filesize=12.0 K 2024-11-14T21:33:53,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~116.29 KB/119080, heapSize ~321.78 KB/329504, currentSize=67.09 KB/68700 for c47d7036b7faae0ba1cfffe0348781b1 in 1569ms, sequenceid=374, compaction requested=false 2024-11-14T21:33:53,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T21:33:53,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T21:33:53,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:53,737 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing c47d7036b7faae0ba1cfffe0348781b1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T21:33:53,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=A 2024-11-14T21:33:53,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:53,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=B 2024-11-14T21:33:53,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:53,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=C 2024-11-14T21:33:53,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:53,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145b12507f8b154d998022f90dff494a59_c47d7036b7faae0ba1cfffe0348781b1 is 50, key is test_row_0/A:col10/1731620032043/Put/seqid=0 2024-11-14T21:33:53,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742214_1390 (size=12454) 2024-11-14T21:33:54,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:54,242 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145b12507f8b154d998022f90dff494a59_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145b12507f8b154d998022f90dff494a59_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:54,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/b1de58db0c154bb0aa56481d08df7d94, store: [table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:54,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/b1de58db0c154bb0aa56481d08df7d94 is 175, key is test_row_0/A:col10/1731620032043/Put/seqid=0 2024-11-14T21:33:54,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742215_1391 (size=31255) 2024-11-14T21:33:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:54,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:54,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620094384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:54,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620094387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:54,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620094389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:54,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620094393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:54,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:54,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620094408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:54,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:54,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620094511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:54,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:54,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620094520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:54,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620094520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:54,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620094520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:54,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:54,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620094540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:54,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T21:33:54,693 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=388, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/b1de58db0c154bb0aa56481d08df7d94 2024-11-14T21:33:54,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/b2df52f20f6847dda056388fba85d9ff is 50, key is test_row_0/B:col10/1731620032043/Put/seqid=0 2024-11-14T21:33:54,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620094731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:54,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742216_1392 (size=12301) 2024-11-14T21:33:54,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:54,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620094731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:54,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620094731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:54,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620094731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:54,749 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/b2df52f20f6847dda056388fba85d9ff 2024-11-14T21:33:54,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:54,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/7222549eae254e61b38d049e15641994 is 50, key is test_row_0/C:col10/1731620032043/Put/seqid=0 2024-11-14T21:33:54,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620094762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:54,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742217_1393 (size=12301) 2024-11-14T21:33:54,818 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/7222549eae254e61b38d049e15641994 2024-11-14T21:33:54,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/b1de58db0c154bb0aa56481d08df7d94 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/b1de58db0c154bb0aa56481d08df7d94 2024-11-14T21:33:54,839 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/b1de58db0c154bb0aa56481d08df7d94, entries=150, sequenceid=388, filesize=30.5 K 2024-11-14T21:33:54,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/b2df52f20f6847dda056388fba85d9ff as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/b2df52f20f6847dda056388fba85d9ff 2024-11-14T21:33:54,853 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/b2df52f20f6847dda056388fba85d9ff, entries=150, sequenceid=388, filesize=12.0 K 2024-11-14T21:33:54,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/7222549eae254e61b38d049e15641994 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/7222549eae254e61b38d049e15641994 2024-11-14T21:33:54,861 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/7222549eae254e61b38d049e15641994, entries=150, sequenceid=388, filesize=12.0 K 2024-11-14T21:33:54,863 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c47d7036b7faae0ba1cfffe0348781b1 in 1126ms, sequenceid=388, compaction requested=true 2024-11-14T21:33:54,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:54,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:54,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-14T21:33:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-14T21:33:54,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-14T21:33:54,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3700 sec 2024-11-14T21:33:54,878 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 2.3890 sec 2024-11-14T21:33:55,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c47d7036b7faae0ba1cfffe0348781b1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T21:33:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=A 2024-11-14T21:33:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=B 2024-11-14T21:33:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=C 2024-11-14T21:33:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:55,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141c50ee358b134fd6a623c304dd690092_c47d7036b7faae0ba1cfffe0348781b1 is 50, key is test_row_0/A:col10/1731620034391/Put/seqid=0 2024-11-14T21:33:55,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620095110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:55,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:55,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620095126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:55,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620095131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620095131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:55,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620095140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:55,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:55,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742218_1394 (size=14994) 2024-11-14T21:33:55,181 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:55,190 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141c50ee358b134fd6a623c304dd690092_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141c50ee358b134fd6a623c304dd690092_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:55,191 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/96c30312bda544b49810ddeb120760ad, store: [table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:55,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/96c30312bda544b49810ddeb120760ad is 175, key is test_row_0/A:col10/1731620034391/Put/seqid=0 2024-11-14T21:33:55,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742219_1395 (size=39949) 2024-11-14T21:33:55,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620095251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:55,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:55,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620095261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:55,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:55,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620095263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:55,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620095264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:55,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:55,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620095271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:55,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:55,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:55,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620095481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:55,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620095481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:55,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:55,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620095481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:55,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:55,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:55,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620095492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:55,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:55,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620095492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:55,633 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=409, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/96c30312bda544b49810ddeb120760ad 2024-11-14T21:33:55,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/bba85e3233c3498b9e38603e9d000780 is 50, key is test_row_0/B:col10/1731620034391/Put/seqid=0 2024-11-14T21:33:55,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742220_1396 (size=12301) 2024-11-14T21:33:55,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620095807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:55,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620095815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:55,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620095815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:55,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:55,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620095815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:55,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:55,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:55,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620095815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:55,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:55,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:55,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:55,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:56,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/bba85e3233c3498b9e38603e9d000780 2024-11-14T21:33:56,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/06eb14938e004a4ebfcd347abb6b11ad is 50, key is test_row_0/C:col10/1731620034391/Put/seqid=0 2024-11-14T21:33:56,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742221_1397 (size=12301) 2024-11-14T21:33:56,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/06eb14938e004a4ebfcd347abb6b11ad 2024-11-14T21:33:56,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/96c30312bda544b49810ddeb120760ad as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/96c30312bda544b49810ddeb120760ad 2024-11-14T21:33:56,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/96c30312bda544b49810ddeb120760ad, entries=200, sequenceid=409, filesize=39.0 K 2024-11-14T21:33:56,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/bba85e3233c3498b9e38603e9d000780 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/bba85e3233c3498b9e38603e9d000780 2024-11-14T21:33:56,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/bba85e3233c3498b9e38603e9d000780, entries=150, sequenceid=409, filesize=12.0 K 2024-11-14T21:33:56,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/06eb14938e004a4ebfcd347abb6b11ad as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/06eb14938e004a4ebfcd347abb6b11ad 2024-11-14T21:33:56,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/06eb14938e004a4ebfcd347abb6b11ad, entries=150, sequenceid=409, filesize=12.0 K 2024-11-14T21:33:56,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for c47d7036b7faae0ba1cfffe0348781b1 in 1128ms, sequenceid=409, compaction requested=true 2024-11-14T21:33:56,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:56,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:33:56,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:56,208 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:33:56,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:33:56,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:56,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:33:56,208 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:33:56,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:56,209 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:33:56,209 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49614 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:33:56,209 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/A is initiating minor compaction (all files) 2024-11-14T21:33:56,209 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/B is initiating minor compaction (all files) 2024-11-14T21:33:56,209 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/A in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:56,209 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/B in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:56,209 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/21dd5fa1c92d46b8a4516e0cbade204c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/31d79eb7e0b941d280d3524dfceed912, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/b1de58db0c154bb0aa56481d08df7d94, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/96c30312bda544b49810ddeb120760ad] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=139.5 K 2024-11-14T21:33:56,209 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/c6b117c1b4d048129c61c33d2c0e4404, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/c3dba67a85b54023931d1542a07267d6, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/b2df52f20f6847dda056388fba85d9ff, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/bba85e3233c3498b9e38603e9d000780] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=48.5 K 2024-11-14T21:33:56,209 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:56,209 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. files: [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/21dd5fa1c92d46b8a4516e0cbade204c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/31d79eb7e0b941d280d3524dfceed912, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/b1de58db0c154bb0aa56481d08df7d94, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/96c30312bda544b49810ddeb120760ad] 2024-11-14T21:33:56,209 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting c6b117c1b4d048129c61c33d2c0e4404, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731620030468 2024-11-14T21:33:56,209 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21dd5fa1c92d46b8a4516e0cbade204c, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731620030468 2024-11-14T21:33:56,210 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting c3dba67a85b54023931d1542a07267d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731620030788 2024-11-14T21:33:56,210 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31d79eb7e0b941d280d3524dfceed912, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731620030788 2024-11-14T21:33:56,210 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting b2df52f20f6847dda056388fba85d9ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731620032043 2024-11-14T21:33:56,210 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1de58db0c154bb0aa56481d08df7d94, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731620032043 2024-11-14T21:33:56,210 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting bba85e3233c3498b9e38603e9d000780, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731620034391 2024-11-14T21:33:56,210 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96c30312bda544b49810ddeb120760ad, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731620034382 2024-11-14T21:33:56,221 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:56,222 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114ecfcb124280644b99ad5eb904ed37dd8_c47d7036b7faae0ba1cfffe0348781b1 store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:56,223 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#B#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:56,224 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/a2cfce99b4cd4aeb8002469ecca58bb4 is 50, key is test_row_0/B:col10/1731620034391/Put/seqid=0 2024-11-14T21:33:56,225 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114ecfcb124280644b99ad5eb904ed37dd8_c47d7036b7faae0ba1cfffe0348781b1, store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:56,225 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ecfcb124280644b99ad5eb904ed37dd8_c47d7036b7faae0ba1cfffe0348781b1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:56,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742223_1399 (size=4469) 2024-11-14T21:33:56,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742222_1398 (size=12847) 2024-11-14T21:33:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:56,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c47d7036b7faae0ba1cfffe0348781b1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T21:33:56,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=A 2024-11-14T21:33:56,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:56,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=B 2024-11-14T21:33:56,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:56,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=C 2024-11-14T21:33:56,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:56,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111411f21a45674e49b29989e42387fa1916_c47d7036b7faae0ba1cfffe0348781b1 is 50, key is test_row_0/A:col10/1731620035126/Put/seqid=0 2024-11-14T21:33:56,378 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T21:33:56,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742224_1400 (size=14994) 2024-11-14T21:33:56,404 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:56,425 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111411f21a45674e49b29989e42387fa1916_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111411f21a45674e49b29989e42387fa1916_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:56,428 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/454f3be101df40b3ae5c92b92f0a60fe, store: [table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:56,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/454f3be101df40b3ae5c92b92f0a60fe is 175, key is test_row_0/A:col10/1731620035126/Put/seqid=0 2024-11-14T21:33:56,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:56,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620096425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:56,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620096450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:56,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:56,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:56,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620096453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:56,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:56,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:56,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620096455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620096456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:56,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742225_1401 (size=39949) 2024-11-14T21:33:56,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:56,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620096567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:56,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:56,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620096581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:56,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620096583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:56,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:56,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620096583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:56,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620096583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:56,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:56,651 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#A#compaction#338 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:56,651 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/8a216107232a4683af29e1f6513a3efc is 175, key is test_row_0/A:col10/1731620034391/Put/seqid=0 2024-11-14T21:33:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T21:33:56,652 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:33:56,653 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:33:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-14T21:33:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-14T21:33:56,663 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:33:56,667 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:33:56,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:33:56,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742226_1402 (size=31801) 2024-11-14T21:33:56,691 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/a2cfce99b4cd4aeb8002469ecca58bb4 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/a2cfce99b4cd4aeb8002469ecca58bb4 2024-11-14T21:33:56,715 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/B of c47d7036b7faae0ba1cfffe0348781b1 into a2cfce99b4cd4aeb8002469ecca58bb4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:56,715 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:56,715 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/B, priority=12, startTime=1731620036208; duration=0sec 2024-11-14T21:33:56,715 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:56,715 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:B 2024-11-14T21:33:56,715 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:33:56,723 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49614 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:33:56,723 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/C is initiating minor compaction (all files) 2024-11-14T21:33:56,723 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/C in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:56,723 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/5ef3f887e4bd445383413faf3f591b50, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/ba7b059c0e5845159c6d029b95cf621f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/7222549eae254e61b38d049e15641994, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/06eb14938e004a4ebfcd347abb6b11ad] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=48.5 K 2024-11-14T21:33:56,726 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ef3f887e4bd445383413faf3f591b50, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731620030468 2024-11-14T21:33:56,730 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting ba7b059c0e5845159c6d029b95cf621f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731620030788 2024-11-14T21:33:56,731 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 7222549eae254e61b38d049e15641994, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731620032043 2024-11-14T21:33:56,731 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 06eb14938e004a4ebfcd347abb6b11ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731620034391 2024-11-14T21:33:56,761 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#C#compaction#341 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:56,762 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/a12bb3609f2a4432bbb1e0e39f8c4290 is 50, key is test_row_0/C:col10/1731620034391/Put/seqid=0 2024-11-14T21:33:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-14T21:33:56,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:56,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620096791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:56,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:56,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620096800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:56,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:56,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620096801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:56,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620096811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:56,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:56,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620096812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:56,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:56,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:56,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742227_1403 (size=12847) 2024-11-14T21:33:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:56,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:33:56,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:56,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:56,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:56,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:56,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:56,842 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/a12bb3609f2a4432bbb1e0e39f8c4290 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/a12bb3609f2a4432bbb1e0e39f8c4290 2024-11-14T21:33:56,851 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/C of c47d7036b7faae0ba1cfffe0348781b1 into a12bb3609f2a4432bbb1e0e39f8c4290(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:56,851 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:56,851 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/C, priority=12, startTime=1731620036208; duration=0sec 2024-11-14T21:33:56,851 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:56,851 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:C 2024-11-14T21:33:56,882 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=421, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/454f3be101df40b3ae5c92b92f0a60fe 2024-11-14T21:33:56,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/5eae9a9e01a34ea6a0b264ba03d2bda5 is 50, key is test_row_0/B:col10/1731620035126/Put/seqid=0 2024-11-14T21:33:56,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742228_1404 (size=12301) 2024-11-14T21:33:56,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/5eae9a9e01a34ea6a0b264ba03d2bda5 2024-11-14T21:33:56,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/3132e357c12d468b8efe714962927606 is 50, key is test_row_0/C:col10/1731620035126/Put/seqid=0 2024-11-14T21:33:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-14T21:33:56,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742229_1405 (size=12301) 2024-11-14T21:33:56,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/3132e357c12d468b8efe714962927606 2024-11-14T21:33:56,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:33:56,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:56,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:56,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:56,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:56,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:56,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/454f3be101df40b3ae5c92b92f0a60fe as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/454f3be101df40b3ae5c92b92f0a60fe 2024-11-14T21:33:56,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/454f3be101df40b3ae5c92b92f0a60fe, entries=200, sequenceid=421, filesize=39.0 K 2024-11-14T21:33:56,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/5eae9a9e01a34ea6a0b264ba03d2bda5 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/5eae9a9e01a34ea6a0b264ba03d2bda5 2024-11-14T21:33:56,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/5eae9a9e01a34ea6a0b264ba03d2bda5, entries=150, sequenceid=421, filesize=12.0 K 2024-11-14T21:33:56,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/3132e357c12d468b8efe714962927606 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/3132e357c12d468b8efe714962927606 2024-11-14T21:33:57,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/3132e357c12d468b8efe714962927606, entries=150, sequenceid=421, filesize=12.0 K 2024-11-14T21:33:57,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for c47d7036b7faae0ba1cfffe0348781b1 in 648ms, sequenceid=421, compaction requested=false 2024-11-14T21:33:57,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:57,096 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/8a216107232a4683af29e1f6513a3efc as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/8a216107232a4683af29e1f6513a3efc 2024-11-14T21:33:57,124 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/A of c47d7036b7faae0ba1cfffe0348781b1 into 8a216107232a4683af29e1f6513a3efc(size=31.1 K), total size for store is 70.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:57,124 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:57,124 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/A, priority=12, startTime=1731620036208; duration=0sec 2024-11-14T21:33:57,124 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:57,124 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:A 2024-11-14T21:33:57,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c47d7036b7faae0ba1cfffe0348781b1 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-14T21:33:57,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=A 2024-11-14T21:33:57,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:57,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=B 2024-11-14T21:33:57,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:57,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=C 2024-11-14T21:33:57,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:57,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:33:57,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:57,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:57,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:57,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:57,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:57,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114071585cac6444beb8f7724b824c10d6e_c47d7036b7faae0ba1cfffe0348781b1 is 50, key is test_row_0/A:col10/1731620037128/Put/seqid=0 2024-11-14T21:33:57,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620097149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:57,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:57,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620097168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:57,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620097171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:57,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:57,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620097173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:57,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620097177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:57,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742230_1406 (size=14994) 2024-11-14T21:33:57,230 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:57,243 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114071585cac6444beb8f7724b824c10d6e_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114071585cac6444beb8f7724b824c10d6e_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:57,244 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/2f2f1c690e644746aefbc2001bb63ec7, store: [table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:57,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/2f2f1c690e644746aefbc2001bb63ec7 is 175, key is test_row_0/A:col10/1731620037128/Put/seqid=0 2024-11-14T21:33:57,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742231_1407 (size=39949) 2024-11-14T21:33:57,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-14T21:33:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620097282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:57,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:57,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:33:57,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:57,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:57,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:57,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:57,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:57,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620097301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:57,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:57,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620097302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:57,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:57,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:57,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620097302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:57,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:57,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620097302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:57,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:57,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:57,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:33:57,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:57,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:57,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:57,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:57,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:57,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620097501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:57,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:57,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620097523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:57,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620097523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:57,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620097533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:57,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620097534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:57,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:57,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:57,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:57,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:57,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:33:57,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:57,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:57,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:57,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:57,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:57,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:57,680 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=446, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/2f2f1c690e644746aefbc2001bb63ec7 2024-11-14T21:33:57,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/821f2427068a4269bee8bb59c19319fd is 50, key is test_row_0/B:col10/1731620037128/Put/seqid=0 2024-11-14T21:33:57,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742232_1408 (size=12301) 2024-11-14T21:33:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-14T21:33:57,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:33:57,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:57,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:57,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:57,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:57,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:57,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:57,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620097840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:57,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:57,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:57,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620097871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:57,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:57,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:57,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620097871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:57,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:57,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:57,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620097871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:57,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:57,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620097871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:57,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:33:57,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:57,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:57,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:57,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:57,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:57,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:58,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:33:58,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:58,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:58,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:58,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:58,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:58,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/821f2427068a4269bee8bb59c19319fd 2024-11-14T21:33:58,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/79e448a645f7405287f9e88b5fca44b9 is 50, key is test_row_0/C:col10/1731620037128/Put/seqid=0 2024-11-14T21:33:58,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742233_1409 (size=12301) 2024-11-14T21:33:58,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/79e448a645f7405287f9e88b5fca44b9 2024-11-14T21:33:58,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/2f2f1c690e644746aefbc2001bb63ec7 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/2f2f1c690e644746aefbc2001bb63ec7 2024-11-14T21:33:58,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/2f2f1c690e644746aefbc2001bb63ec7, entries=200, sequenceid=446, filesize=39.0 K 2024-11-14T21:33:58,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/821f2427068a4269bee8bb59c19319fd as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/821f2427068a4269bee8bb59c19319fd 2024-11-14T21:33:58,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:33:58,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:58,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:58,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:58,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:58,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:58,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/821f2427068a4269bee8bb59c19319fd, entries=150, sequenceid=446, filesize=12.0 K 2024-11-14T21:33:58,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/79e448a645f7405287f9e88b5fca44b9 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/79e448a645f7405287f9e88b5fca44b9 2024-11-14T21:33:58,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/79e448a645f7405287f9e88b5fca44b9, entries=150, sequenceid=446, filesize=12.0 K 2024-11-14T21:33:58,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=53.67 KB/54960 for c47d7036b7faae0ba1cfffe0348781b1 in 1178ms, sequenceid=446, compaction requested=true 2024-11-14T21:33:58,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:58,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:33:58,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:58,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:33:58,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:33:58,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:33:58,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T21:33:58,310 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:58,310 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:58,320 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:58,320 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:58,320 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/C is initiating minor compaction (all files) 2024-11-14T21:33:58,320 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/A is initiating minor compaction (all files) 2024-11-14T21:33:58,320 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/C in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:58,320 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/A in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:58,320 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/a12bb3609f2a4432bbb1e0e39f8c4290, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/3132e357c12d468b8efe714962927606, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/79e448a645f7405287f9e88b5fca44b9] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=36.6 K 2024-11-14T21:33:58,320 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/8a216107232a4683af29e1f6513a3efc, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/454f3be101df40b3ae5c92b92f0a60fe, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/2f2f1c690e644746aefbc2001bb63ec7] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=109.1 K 2024-11-14T21:33:58,320 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:58,320 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. files: [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/8a216107232a4683af29e1f6513a3efc, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/454f3be101df40b3ae5c92b92f0a60fe, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/2f2f1c690e644746aefbc2001bb63ec7] 2024-11-14T21:33:58,323 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a216107232a4683af29e1f6513a3efc, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731620034391 2024-11-14T21:33:58,323 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting a12bb3609f2a4432bbb1e0e39f8c4290, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731620034391 2024-11-14T21:33:58,330 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 454f3be101df40b3ae5c92b92f0a60fe, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731620035118 2024-11-14T21:33:58,330 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 3132e357c12d468b8efe714962927606, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731620035118 2024-11-14T21:33:58,332 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f2f1c690e644746aefbc2001bb63ec7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731620036421 2024-11-14T21:33:58,334 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 79e448a645f7405287f9e88b5fca44b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731620036421 2024-11-14T21:33:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:58,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c47d7036b7faae0ba1cfffe0348781b1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T21:33:58,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=A 2024-11-14T21:33:58,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:58,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=B 2024-11-14T21:33:58,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:58,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=C 2024-11-14T21:33:58,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:33:58,377 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:58,384 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#C#compaction#348 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:58,385 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/28ef034d844a44f0853730a2c7a8ced7 is 50, key is test_row_0/C:col10/1731620037128/Put/seqid=0 2024-11-14T21:33:58,387 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111435fd235820844ef797140ec0e9c713c7_c47d7036b7faae0ba1cfffe0348781b1 store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:58,390 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111435fd235820844ef797140ec0e9c713c7_c47d7036b7faae0ba1cfffe0348781b1, store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:58,390 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111435fd235820844ef797140ec0e9c713c7_c47d7036b7faae0ba1cfffe0348781b1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:58,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ec26c149e9bf4afaaf13ac4a1eec3e1b_c47d7036b7faae0ba1cfffe0348781b1 is 50, key is test_row_0/A:col10/1731620037140/Put/seqid=0 2024-11-14T21:33:58,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:33:58,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:58,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:58,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:58,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:58,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:58,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742235_1411 (size=4469) 2024-11-14T21:33:58,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742234_1410 (size=12949) 2024-11-14T21:33:58,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:58,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620098465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:58,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:58,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742236_1412 (size=14994) 2024-11-14T21:33:58,485 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:58,489 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ec26c149e9bf4afaaf13ac4a1eec3e1b_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ec26c149e9bf4afaaf13ac4a1eec3e1b_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:58,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:58,491 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/4400eda890ff454f96c35e9e227db91e, store: [table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:58,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/4400eda890ff454f96c35e9e227db91e is 175, key is test_row_0/A:col10/1731620037140/Put/seqid=0 2024-11-14T21:33:58,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620098472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:58,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:58,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:58,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620098476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:58,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620098481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:58,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:58,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620098483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:58,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:58,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742237_1413 (size=39949) 2024-11-14T21:33:58,534 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=458, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/4400eda890ff454f96c35e9e227db91e 2024-11-14T21:33:58,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/5e8cfbdcde4e4a90a9d571798cac2530 is 50, key is test_row_0/B:col10/1731620037140/Put/seqid=0 2024-11-14T21:33:58,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742238_1414 (size=12301) 2024-11-14T21:33:58,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/5e8cfbdcde4e4a90a9d571798cac2530 2024-11-14T21:33:58,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:33:58,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:58,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:58,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:58,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:58,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:58,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:58,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/ba52d0eaa3f241eb88f02eadc2142291 is 50, key is test_row_0/C:col10/1731620037140/Put/seqid=0 2024-11-14T21:33:58,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620098583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:58,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:58,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620098601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:58,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:58,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620098602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:58,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:58,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620098603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:58,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:58,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:58,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:58,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620098611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:58,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742239_1415 (size=12301) 2024-11-14T21:33:58,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:33:58,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:58,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:58,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:58,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:58,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:58,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-14T21:33:58,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620098821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:58,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:58,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620098821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:58,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:58,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620098822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:58,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620098833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:58,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:58,876 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#A#compaction#347 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:58,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620098859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:58,877 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/485be28bdf9a478281cb272cbba558d8 is 175, key is test_row_0/A:col10/1731620037128/Put/seqid=0 2024-11-14T21:33:58,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:58,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:58,889 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/28ef034d844a44f0853730a2c7a8ced7 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/28ef034d844a44f0853730a2c7a8ced7 2024-11-14T21:33:58,896 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/C of c47d7036b7faae0ba1cfffe0348781b1 into 28ef034d844a44f0853730a2c7a8ced7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:58,896 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:58,896 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/C, priority=13, startTime=1731620038310; duration=0sec 2024-11-14T21:33:58,896 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:33:58,896 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:C 2024-11-14T21:33:58,896 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:33:58,897 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:33:58,897 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/B is initiating minor compaction (all files) 2024-11-14T21:33:58,897 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/B in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:58,897 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/a2cfce99b4cd4aeb8002469ecca58bb4, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/5eae9a9e01a34ea6a0b264ba03d2bda5, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/821f2427068a4269bee8bb59c19319fd] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=36.6 K 2024-11-14T21:33:58,898 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting a2cfce99b4cd4aeb8002469ecca58bb4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731620034391 2024-11-14T21:33:58,899 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 5eae9a9e01a34ea6a0b264ba03d2bda5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731620035118 2024-11-14T21:33:58,900 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 821f2427068a4269bee8bb59c19319fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731620036421 2024-11-14T21:33:58,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:33:58,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:58,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:58,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:58,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:58,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:58,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742240_1416 (size=31903) 2024-11-14T21:33:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:58,917 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#B#compaction#352 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:33:58,918 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/f24a89a7ba614b80b72f57aebd12160e is 50, key is test_row_0/B:col10/1731620037128/Put/seqid=0 2024-11-14T21:33:58,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742241_1417 (size=12949) 2024-11-14T21:33:59,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/ba52d0eaa3f241eb88f02eadc2142291 2024-11-14T21:33:59,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:33:59,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:59,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:59,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:59,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:59,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:59,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:59,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/4400eda890ff454f96c35e9e227db91e as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/4400eda890ff454f96c35e9e227db91e 2024-11-14T21:33:59,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/4400eda890ff454f96c35e9e227db91e, entries=200, sequenceid=458, filesize=39.0 K 2024-11-14T21:33:59,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/5e8cfbdcde4e4a90a9d571798cac2530 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/5e8cfbdcde4e4a90a9d571798cac2530 2024-11-14T21:33:59,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:59,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620099140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:59,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/5e8cfbdcde4e4a90a9d571798cac2530, entries=150, sequenceid=458, filesize=12.0 K 2024-11-14T21:33:59,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:59,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620099151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:59,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:59,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:59,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:59,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:59,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620099151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:59,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:59,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:59,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:59,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/ba52d0eaa3f241eb88f02eadc2142291 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/ba52d0eaa3f241eb88f02eadc2142291 2024-11-14T21:33:59,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:33:59,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620099160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:33:59,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:59,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:33:59,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:33:59,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/ba52d0eaa3f241eb88f02eadc2142291, entries=150, sequenceid=458, filesize=12.0 K 2024-11-14T21:33:59,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c47d7036b7faae0ba1cfffe0348781b1 in 809ms, sequenceid=458, compaction requested=false 2024-11-14T21:33:59,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:59,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:59,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731620038463, max timestamp=1731620039190Num uniques 859; 2024-11-14T21:33:59,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731620038463, max timestamp=1731620039190Num uniques 859; 2024-11-14T21:33:59,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:33:59,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731620038463, max timestamp=1731620039190Num uniques 859; 2024-11-14T21:33:59,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:33:59,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:33:59,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c47d7036b7faae0ba1cfffe0348781b1 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-11-14T21:33:59,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=A 2024-11-14T21:33:59,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:59,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=B 2024-11-14T21:33:59,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:59,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=C 2024-11-14T21:33:59,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:33:59,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114730f99cb35fb4da1bfe34c30ceab5a9d_c47d7036b7faae0ba1cfffe0348781b1 is 50, key is test_row_0/A:col10/1731620038463/Put/seqid=0 2024-11-14T21:33:59,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742242_1418 (size=14994) 2024-11-14T21:33:59,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:33:59,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:59,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:59,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:59,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:59,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:59,231 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:33:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:59,242 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114730f99cb35fb4da1bfe34c30ceab5a9d_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114730f99cb35fb4da1bfe34c30ceab5a9d_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:33:59,243 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/6b071b9c1ea24e11ae0732605ed3af92, store: [table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:33:59,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/6b071b9c1ea24e11ae0732605ed3af92 is 175, key is test_row_0/A:col10/1731620038463/Put/seqid=0 2024-11-14T21:33:59,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742243_1419 (size=39949) 2024-11-14T21:33:59,323 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/485be28bdf9a478281cb272cbba558d8 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/485be28bdf9a478281cb272cbba558d8 2024-11-14T21:33:59,329 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/A of c47d7036b7faae0ba1cfffe0348781b1 into 485be28bdf9a478281cb272cbba558d8(size=31.2 K), total size for store is 70.2 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-14T21:33:59,329 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:59,329 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/A, priority=13, startTime=1731620038310; duration=1sec 2024-11-14T21:33:59,329 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:59,329 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:A 2024-11-14T21:33:59,345 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/f24a89a7ba614b80b72f57aebd12160e as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/f24a89a7ba614b80b72f57aebd12160e 2024-11-14T21:33:59,352 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/B of c47d7036b7faae0ba1cfffe0348781b1 into f24a89a7ba614b80b72f57aebd12160e(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:33:59,352 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:33:59,352 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/B, priority=13, startTime=1731620038310; duration=0sec 2024-11-14T21:33:59,352 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:33:59,352 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:B 2024-11-14T21:33:59,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:33:59,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731620039347, max timestamp=1731620039389Num uniques 859; 2024-11-14T21:33:59,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731620039347, max timestamp=1731620039389Num uniques 859; 2024-11-14T21:33:59,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:33:59,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:33:59,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:59,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:59,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:59,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:59,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:59,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731620039347, max timestamp=1731620039389Num uniques 859; 2024-11-14T21:33:59,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:33:59,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:33:59,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:59,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:59,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:59,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:59,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:59,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731620039525, max timestamp=1731620039606Num uniques 1018; 2024-11-14T21:33:59,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T21:33:59,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731620039525, max timestamp=1731620039606Num uniques 1018; 2024-11-14T21:33:59,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T21:33:59,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731620039525, max timestamp=1731620039606Num uniques 1018; 2024-11-14T21:33:59,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T21:33:59,656 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=480, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/6b071b9c1ea24e11ae0732605ed3af92 2024-11-14T21:33:59,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/f649b84eaef94fd7b802560c7f952ac8 is 50, key is test_row_0/B:col10/1731620038463/Put/seqid=0 2024-11-14T21:33:59,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742244_1420 (size=12301) 2024-11-14T21:33:59,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:33:59,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:59,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:59,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:59,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:59,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:59,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:59,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731620039698, max timestamp=1731620039725Num uniques 1027; 2024-11-14T21:33:59,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-14T21:33:59,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731620039698, max timestamp=1731620039725Num uniques 1027; 2024-11-14T21:33:59,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731620039698, max timestamp=1731620039725Num uniques 1027; 2024-11-14T21:33:59,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-14T21:33:59,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-14T21:33:59,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731620039784, max timestamp=1731620039819Num uniques 1086; 2024-11-14T21:33:59,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-11-14T21:33:59,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731620039784, max timestamp=1731620039820Num uniques 1086; 2024-11-14T21:33:59,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-11-14T21:33:59,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731620039784, max timestamp=1731620039820Num uniques 1086; 2024-11-14T21:33:59,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-11-14T21:33:59,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:33:59,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:59,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:33:59,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:33:59,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:59,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:33:59,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731620039890, max timestamp=1731620039928Num uniques 1195; 2024-11-14T21:33:59,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=13510, suffix heap size=152248,new segment heap size=30920  suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=295 2024-11-14T21:33:59,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731620039890, max timestamp=1731620039928Num uniques 1195; 2024-11-14T21:33:59,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731620039890, max timestamp=1731620039928Num uniques 1195; 2024-11-14T21:33:59,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=13510, suffix heap size=152248,new segment heap size=30920  suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=295 2024-11-14T21:33:59,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=13510, suffix heap size=152248,new segment heap size=30920  suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=295 2024-11-14T21:34:00,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:34:00,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:00,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:34:00,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:00,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:00,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:00,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620099999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:00,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:00,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1731620039991, max timestamp=1731620040021Num uniques 1154; 2024-11-14T21:34:00,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=13920, suffix heap size=147984,new segment heap size=31856  suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=304 2024-11-14T21:34:00,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1731620039991, max timestamp=1731620040021Num uniques 1154; 2024-11-14T21:34:00,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=13920, suffix heap size=147984,new segment heap size=31856  suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=304 2024-11-14T21:34:00,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1731620039991, max timestamp=1731620040021Num uniques 1154; 2024-11-14T21:34:00,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=13920, suffix heap size=147984,new segment heap size=31856  suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=304 2024-11-14T21:34:00,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/f649b84eaef94fd7b802560c7f952ac8 2024-11-14T21:34:00,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/0fd7d12209334beda3eb33948bab45ec is 50, key is test_row_0/C:col10/1731620038463/Put/seqid=0 2024-11-14T21:34:00,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742245_1421 (size=12301) 2024-11-14T21:34:00,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/0fd7d12209334beda3eb33948bab45ec 2024-11-14T21:34:00,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/6b071b9c1ea24e11ae0732605ed3af92 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/6b071b9c1ea24e11ae0732605ed3af92 2024-11-14T21:34:00,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620100107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:00,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620100107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:00,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620100108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:00,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731620040083, max timestamp=1731620040124Num uniques 1163; 2024-11-14T21:34:00,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=7458, suffix heap size=148920,new segment heap size=17192  suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=163 2024-11-14T21:34:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:00,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/6b071b9c1ea24e11ae0732605ed3af92, entries=200, sequenceid=480, filesize=39.0 K 2024-11-14T21:34:00,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731620040083, max timestamp=1731620040124Num uniques 1163; 2024-11-14T21:34:00,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=7458, suffix heap size=148920,new segment heap size=17192  suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=163 2024-11-14T21:34:00,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731620040083, max timestamp=1731620040124Num uniques 1163; 2024-11-14T21:34:00,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=7458, suffix heap size=148920,new segment heap size=17192  suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=163 2024-11-14T21:34:00,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/f649b84eaef94fd7b802560c7f952ac8 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/f649b84eaef94fd7b802560c7f952ac8 2024-11-14T21:34:00,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/f649b84eaef94fd7b802560c7f952ac8, entries=150, sequenceid=480, filesize=12.0 K 2024-11-14T21:34:00,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/0fd7d12209334beda3eb33948bab45ec as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/0fd7d12209334beda3eb33948bab45ec 2024-11-14T21:34:00,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/0fd7d12209334beda3eb33948bab45ec, entries=150, sequenceid=480, filesize=12.0 K 2024-11-14T21:34:00,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=53.67 KB/54960 for c47d7036b7faae0ba1cfffe0348781b1 in 963ms, sequenceid=480, compaction requested=true 2024-11-14T21:34:00,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:34:00,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:34:00,163 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:00,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:00,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:34:00,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:00,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:34:00,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:00,163 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:00,164 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111801 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:00,164 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/A is initiating minor compaction (all files) 2024-11-14T21:34:00,164 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/A in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:00,164 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/485be28bdf9a478281cb272cbba558d8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/4400eda890ff454f96c35e9e227db91e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/6b071b9c1ea24e11ae0732605ed3af92] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=109.2 K 2024-11-14T21:34:00,164 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:00,164 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. files: [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/485be28bdf9a478281cb272cbba558d8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/4400eda890ff454f96c35e9e227db91e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/6b071b9c1ea24e11ae0732605ed3af92] 2024-11-14T21:34:00,164 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 485be28bdf9a478281cb272cbba558d8, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731620036421 2024-11-14T21:34:00,165 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4400eda890ff454f96c35e9e227db91e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731620037140 2024-11-14T21:34:00,165 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b071b9c1ea24e11ae0732605ed3af92, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731620038463 2024-11-14T21:34:00,166 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:00,166 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/B is initiating minor compaction (all files) 2024-11-14T21:34:00,166 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/B in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:00,166 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/f24a89a7ba614b80b72f57aebd12160e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/5e8cfbdcde4e4a90a9d571798cac2530, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/f649b84eaef94fd7b802560c7f952ac8] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=36.7 K 2024-11-14T21:34:00,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:00,167 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting f24a89a7ba614b80b72f57aebd12160e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731620036421 2024-11-14T21:34:00,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c47d7036b7faae0ba1cfffe0348781b1 3/3 column families, dataSize=60.38 KB heapSize=165.47 KB 2024-11-14T21:34:00,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=A 2024-11-14T21:34:00,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:00,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=B 2024-11-14T21:34:00,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:00,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=C 2024-11-14T21:34:00,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:00,168 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e8cfbdcde4e4a90a9d571798cac2530, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731620037140 2024-11-14T21:34:00,172 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting f649b84eaef94fd7b802560c7f952ac8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731620038463 2024-11-14T21:34:00,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:34:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:34:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:00,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:00,181 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:34:00,185 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#B#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:00,185 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/e351c0d37889437593348da04c0fc19f is 50, key is test_row_0/B:col10/1731620038463/Put/seqid=0 2024-11-14T21:34:00,187 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114cd7fc6a6f69e4706833407f8cf83c0dc_c47d7036b7faae0ba1cfffe0348781b1 store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:34:00,189 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114cd7fc6a6f69e4706833407f8cf83c0dc_c47d7036b7faae0ba1cfffe0348781b1, store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:34:00,189 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cd7fc6a6f69e4706833407f8cf83c0dc_c47d7036b7faae0ba1cfffe0348781b1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:34:00,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b181de63c7f14ec698ed454c2f2c0de4_c47d7036b7faae0ba1cfffe0348781b1 is 50, key is test_row_0/A:col10/1731620040083/Put/seqid=0 2024-11-14T21:34:00,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742246_1422 (size=13051) 2024-11-14T21:34:00,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742247_1423 (size=4469) 2024-11-14T21:34:00,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742248_1424 (size=14994) 2024-11-14T21:34:00,271 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#A#compaction#356 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:00,272 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/ba468eae760c4ddb995952c4594201b8 is 175, key is test_row_0/A:col10/1731620038463/Put/seqid=0 2024-11-14T21:34:00,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742249_1425 (size=32005) 2024-11-14T21:34:00,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620100283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:00,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731620040235, max timestamp=1731620040301Num uniques 859; 2024-11-14T21:34:00,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T21:34:00,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731620040235, max timestamp=1731620040301Num uniques 859; 2024-11-14T21:34:00,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T21:34:00,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731620040235, max timestamp=1731620040301Num uniques 859; 2024-11-14T21:34:00,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T21:34:00,317 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/ba468eae760c4ddb995952c4594201b8 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/ba468eae760c4ddb995952c4594201b8 2024-11-14T21:34:00,328 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/A of c47d7036b7faae0ba1cfffe0348781b1 into ba468eae760c4ddb995952c4594201b8(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:00,329 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:34:00,329 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/A, priority=13, startTime=1731620040162; duration=0sec 2024-11-14T21:34:00,329 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:00,329 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:A 2024-11-14T21:34:00,329 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:00,330 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:00,330 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/C is initiating minor compaction (all files) 2024-11-14T21:34:00,330 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/C in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:00,330 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/28ef034d844a44f0853730a2c7a8ced7, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/ba52d0eaa3f241eb88f02eadc2142291, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/0fd7d12209334beda3eb33948bab45ec] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=36.7 K 2024-11-14T21:34:00,331 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28ef034d844a44f0853730a2c7a8ced7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731620036421 2024-11-14T21:34:00,331 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba52d0eaa3f241eb88f02eadc2142291, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731620037140 2024-11-14T21:34:00,332 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fd7d12209334beda3eb33948bab45ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731620038463 2024-11-14T21:34:00,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:34:00,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:00,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:34:00,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:00,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:00,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:00,353 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#C#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:00,354 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/16de88725b674d2ca719dc770a91e6b4 is 50, key is test_row_0/C:col10/1731620038463/Put/seqid=0 2024-11-14T21:34:00,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742250_1426 (size=13051) 2024-11-14T21:34:00,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:00,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620100379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:00,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:00,388 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/16de88725b674d2ca719dc770a91e6b4 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/16de88725b674d2ca719dc770a91e6b4 2024-11-14T21:34:00,392 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/C of c47d7036b7faae0ba1cfffe0348781b1 into 16de88725b674d2ca719dc770a91e6b4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:00,392 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:34:00,392 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/C, priority=13, startTime=1731620040163; duration=0sec 2024-11-14T21:34:00,392 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:00,392 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:C 2024-11-14T21:34:00,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:00,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620100380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:00,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:00,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620100381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:00,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:00,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:00,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:00,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620100382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:00,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:00,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620100410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:00,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:00,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:00,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620100490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:00,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:00,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:34:00,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:00,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:34:00,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:00,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:00,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:00,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:00,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620100510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:00,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:00,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620100510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:00,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:00,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620100511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:00,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:00,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:00,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620100623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:00,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:00,643 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/e351c0d37889437593348da04c0fc19f as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/e351c0d37889437593348da04c0fc19f 2024-11-14T21:34:00,650 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/B of c47d7036b7faae0ba1cfffe0348781b1 into e351c0d37889437593348da04c0fc19f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:00,650 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:34:00,650 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/B, priority=13, startTime=1731620040163; duration=0sec 2024-11-14T21:34:00,650 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:00,650 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:B 2024-11-14T21:34:00,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:34:00,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:00,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:34:00,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:00,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:00,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:00,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:00,672 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:00,681 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b181de63c7f14ec698ed454c2f2c0de4_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b181de63c7f14ec698ed454c2f2c0de4_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:00,683 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/1cbf3b6a8c7c4936b9b2f072c8a20f19, store: [table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:34:00,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/1cbf3b6a8c7c4936b9b2f072c8a20f19 is 175, key is test_row_0/A:col10/1731620040083/Put/seqid=0 2024-11-14T21:34:00,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620100711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:00,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:00,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742251_1427 (size=39949) 2024-11-14T21:34:00,736 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=611, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/1cbf3b6a8c7c4936b9b2f072c8a20f19 2024-11-14T21:34:00,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620100731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:00,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620100731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:00,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620100731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:00,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/ea41a6e6b654428f9cf96362239c78ec is 50, key is test_row_0/B:col10/1731620040083/Put/seqid=0 2024-11-14T21:34:00,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742252_1428 (size=12301) 2024-11-14T21:34:00,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:34:00,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:00,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:34:00,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:00,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:00,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:00,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=611 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/ea41a6e6b654428f9cf96362239c78ec 2024-11-14T21:34:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:00,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/96928cffe2c14a82a305a1c1441c3795 is 50, key is test_row_0/C:col10/1731620040083/Put/seqid=0 2024-11-14T21:34:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-14T21:34:00,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742253_1429 (size=12301) 2024-11-14T21:34:00,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620100951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:00,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:34:00,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:00,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:34:00,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:00,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:00,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:00,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:01,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:01,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620101041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:01,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620101050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:01,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620101050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:01,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620101050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:01,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:01,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:34:01,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:01,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:34:01,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:01,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:01,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:01,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:01,147 WARN [regionserver/ef788a4cade8:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-14T21:34:01,151 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/2bc45d594cd9436c8d5dec600893b948, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/50eace1131ea4973ae9700b10ed2f41e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/dcb4d04957aa45dd851d316c885db620, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/df0c14b241e64664bc00b7de147ba207, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/c7f88ae6aa05447b9ec27eeaee3e8193, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/e4f7118d1186475aa4266d77af35c471, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/db941d86545e4188893d73292fbae917, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/af01915940134012bf0ed52f9c00ceb8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/9faa843d05614d9da4e5a92a8794def1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/e00f83f2eec24d258ef81dd69b00fc67, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/ca40ba5f86dd4c048d26b0f99fcf2d82, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/2bdd0a8d5746431887a773856ad590d1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/21dd5fa1c92d46b8a4516e0cbade204c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/31d79eb7e0b941d280d3524dfceed912, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/b1de58db0c154bb0aa56481d08df7d94, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/96c30312bda544b49810ddeb120760ad, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/8a216107232a4683af29e1f6513a3efc, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/454f3be101df40b3ae5c92b92f0a60fe, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/2f2f1c690e644746aefbc2001bb63ec7, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/485be28bdf9a478281cb272cbba558d8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/4400eda890ff454f96c35e9e227db91e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/6b071b9c1ea24e11ae0732605ed3af92] to archive 2024-11-14T21:34:01,158 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:34:01,164 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/2bc45d594cd9436c8d5dec600893b948 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/2bc45d594cd9436c8d5dec600893b948 2024-11-14T21:34:01,173 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/50eace1131ea4973ae9700b10ed2f41e to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/50eace1131ea4973ae9700b10ed2f41e 2024-11-14T21:34:01,177 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/dcb4d04957aa45dd851d316c885db620 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/dcb4d04957aa45dd851d316c885db620 2024-11-14T21:34:01,187 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/df0c14b241e64664bc00b7de147ba207 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/df0c14b241e64664bc00b7de147ba207 2024-11-14T21:34:01,188 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/c7f88ae6aa05447b9ec27eeaee3e8193 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/c7f88ae6aa05447b9ec27eeaee3e8193 2024-11-14T21:34:01,189 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/e4f7118d1186475aa4266d77af35c471 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/e4f7118d1186475aa4266d77af35c471 2024-11-14T21:34:01,192 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/db941d86545e4188893d73292fbae917 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/db941d86545e4188893d73292fbae917 2024-11-14T21:34:01,197 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/af01915940134012bf0ed52f9c00ceb8 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/af01915940134012bf0ed52f9c00ceb8 2024-11-14T21:34:01,206 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/9faa843d05614d9da4e5a92a8794def1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/9faa843d05614d9da4e5a92a8794def1 2024-11-14T21:34:01,210 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/e00f83f2eec24d258ef81dd69b00fc67 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/e00f83f2eec24d258ef81dd69b00fc67 2024-11-14T21:34:01,216 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/ca40ba5f86dd4c048d26b0f99fcf2d82 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/ca40ba5f86dd4c048d26b0f99fcf2d82 2024-11-14T21:34:01,228 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/2bdd0a8d5746431887a773856ad590d1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/2bdd0a8d5746431887a773856ad590d1 2024-11-14T21:34:01,235 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/21dd5fa1c92d46b8a4516e0cbade204c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/21dd5fa1c92d46b8a4516e0cbade204c 2024-11-14T21:34:01,236 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/31d79eb7e0b941d280d3524dfceed912 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/31d79eb7e0b941d280d3524dfceed912 2024-11-14T21:34:01,237 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/b1de58db0c154bb0aa56481d08df7d94 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/b1de58db0c154bb0aa56481d08df7d94 2024-11-14T21:34:01,238 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/96c30312bda544b49810ddeb120760ad to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/96c30312bda544b49810ddeb120760ad 2024-11-14T21:34:01,240 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/8a216107232a4683af29e1f6513a3efc to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/8a216107232a4683af29e1f6513a3efc 2024-11-14T21:34:01,241 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/454f3be101df40b3ae5c92b92f0a60fe to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/454f3be101df40b3ae5c92b92f0a60fe 2024-11-14T21:34:01,242 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/2f2f1c690e644746aefbc2001bb63ec7 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/2f2f1c690e644746aefbc2001bb63ec7 2024-11-14T21:34:01,243 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/485be28bdf9a478281cb272cbba558d8 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/485be28bdf9a478281cb272cbba558d8 2024-11-14T21:34:01,244 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/4400eda890ff454f96c35e9e227db91e to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/4400eda890ff454f96c35e9e227db91e 2024-11-14T21:34:01,245 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/6b071b9c1ea24e11ae0732605ed3af92 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/6b071b9c1ea24e11ae0732605ed3af92 2024-11-14T21:34:01,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=611 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/96928cffe2c14a82a305a1c1441c3795 2024-11-14T21:34:01,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/1cbf3b6a8c7c4936b9b2f072c8a20f19 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/1cbf3b6a8c7c4936b9b2f072c8a20f19 2024-11-14T21:34:01,260 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/762cd60921b64d95a667d8f6fb4ab443, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/9bab9952a7634cef90dfd7ec8d311f76, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/00ce6fe3e6c0436185946eea3a67aed8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/35ef74964131426097183d5ed3c870b3, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/133c3a15ff7c482093ec134fd4e19b3e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/a62b8dd604ef47f4ba9a7bce7052eccd, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/d9aefa8d9bef408493f1cf4a523af04d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/1b08e0ab86754302a99cc8826d8c166d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/70d816227f034568a0eb2d0a69281ddb, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/b8f22d5ea4cb4fe89bcff1c7668cd013, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/771bae3374ea4d499c97eec94d91ad4b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/c6b117c1b4d048129c61c33d2c0e4404, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/d3d046faa0724dd29de9102a76cf85d8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/c3dba67a85b54023931d1542a07267d6, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/b2df52f20f6847dda056388fba85d9ff, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/a2cfce99b4cd4aeb8002469ecca58bb4, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/bba85e3233c3498b9e38603e9d000780, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/5eae9a9e01a34ea6a0b264ba03d2bda5, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/f24a89a7ba614b80b72f57aebd12160e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/821f2427068a4269bee8bb59c19319fd, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/5e8cfbdcde4e4a90a9d571798cac2530, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/f649b84eaef94fd7b802560c7f952ac8] to archive 2024-11-14T21:34:01,261 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:34:01,263 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/762cd60921b64d95a667d8f6fb4ab443 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/762cd60921b64d95a667d8f6fb4ab443 2024-11-14T21:34:01,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/1cbf3b6a8c7c4936b9b2f072c8a20f19, entries=200, sequenceid=611, filesize=39.0 K 2024-11-14T21:34:01,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/ea41a6e6b654428f9cf96362239c78ec as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/ea41a6e6b654428f9cf96362239c78ec 2024-11-14T21:34:01,266 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/9bab9952a7634cef90dfd7ec8d311f76 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/9bab9952a7634cef90dfd7ec8d311f76 2024-11-14T21:34:01,268 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/00ce6fe3e6c0436185946eea3a67aed8 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/00ce6fe3e6c0436185946eea3a67aed8 2024-11-14T21:34:01,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/ea41a6e6b654428f9cf96362239c78ec, entries=150, sequenceid=611, filesize=12.0 K 2024-11-14T21:34:01,269 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/35ef74964131426097183d5ed3c870b3 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/35ef74964131426097183d5ed3c870b3 2024-11-14T21:34:01,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/96928cffe2c14a82a305a1c1441c3795 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/96928cffe2c14a82a305a1c1441c3795 2024-11-14T21:34:01,271 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/133c3a15ff7c482093ec134fd4e19b3e to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/133c3a15ff7c482093ec134fd4e19b3e 2024-11-14T21:34:01,273 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/a62b8dd604ef47f4ba9a7bce7052eccd to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/a62b8dd604ef47f4ba9a7bce7052eccd 2024-11-14T21:34:01,282 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/d9aefa8d9bef408493f1cf4a523af04d to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/d9aefa8d9bef408493f1cf4a523af04d 2024-11-14T21:34:01,284 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/1b08e0ab86754302a99cc8826d8c166d to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/1b08e0ab86754302a99cc8826d8c166d 2024-11-14T21:34:01,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/96928cffe2c14a82a305a1c1441c3795, entries=150, sequenceid=611, filesize=12.0 K 2024-11-14T21:34:01,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T21:34:01,291 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/70d816227f034568a0eb2d0a69281ddb to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/70d816227f034568a0eb2d0a69281ddb 2024-11-14T21:34:01,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~165.42 KB/169392, currentSize=127.47 KB/130530 for c47d7036b7faae0ba1cfffe0348781b1 in 1123ms, sequenceid=611, compaction requested=false 2024-11-14T21:34:01,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:01,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:34:01,291 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing c47d7036b7faae0ba1cfffe0348781b1 3/3 column families, dataSize=127.47 KB heapSize=351 KB 2024-11-14T21:34:01,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=A 2024-11-14T21:34:01,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:01,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=B 2024-11-14T21:34:01,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:01,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=C 2024-11-14T21:34:01,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:01,299 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/b8f22d5ea4cb4fe89bcff1c7668cd013 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/b8f22d5ea4cb4fe89bcff1c7668cd013 2024-11-14T21:34:01,311 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/771bae3374ea4d499c97eec94d91ad4b to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/771bae3374ea4d499c97eec94d91ad4b 2024-11-14T21:34:01,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144b3dd97fae2f4b10a271a66c91bb6891_c47d7036b7faae0ba1cfffe0348781b1 is 50, key is test_row_0/A:col10/1731620040355/Put/seqid=0 2024-11-14T21:34:01,315 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/c6b117c1b4d048129c61c33d2c0e4404 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/c6b117c1b4d048129c61c33d2c0e4404 2024-11-14T21:34:01,325 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/d3d046faa0724dd29de9102a76cf85d8 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/d3d046faa0724dd29de9102a76cf85d8 2024-11-14T21:34:01,327 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/c3dba67a85b54023931d1542a07267d6 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/c3dba67a85b54023931d1542a07267d6 2024-11-14T21:34:01,340 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/b2df52f20f6847dda056388fba85d9ff to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/b2df52f20f6847dda056388fba85d9ff 2024-11-14T21:34:01,347 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/a2cfce99b4cd4aeb8002469ecca58bb4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/a2cfce99b4cd4aeb8002469ecca58bb4 2024-11-14T21:34:01,348 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/bba85e3233c3498b9e38603e9d000780 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/bba85e3233c3498b9e38603e9d000780 2024-11-14T21:34:01,355 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/5eae9a9e01a34ea6a0b264ba03d2bda5 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/5eae9a9e01a34ea6a0b264ba03d2bda5 2024-11-14T21:34:01,359 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/f24a89a7ba614b80b72f57aebd12160e to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/f24a89a7ba614b80b72f57aebd12160e 2024-11-14T21:34:01,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742254_1430 (size=12454) 2024-11-14T21:34:01,363 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/821f2427068a4269bee8bb59c19319fd to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/821f2427068a4269bee8bb59c19319fd 2024-11-14T21:34:01,375 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/5e8cfbdcde4e4a90a9d571798cac2530 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/5e8cfbdcde4e4a90a9d571798cac2530 2024-11-14T21:34:01,382 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/f649b84eaef94fd7b802560c7f952ac8 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/f649b84eaef94fd7b802560c7f952ac8 2024-11-14T21:34:01,387 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/795ecb3d8808428fbb09b6a9d498419d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/a21ee0eafcae4050a49997f913e1a4e1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/cf78ff02093e49a383d8505ba7a2b583, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/e4d2d0b18ff64cf18c9621b264549460, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/2fdce2cb2fea4d46803ad70ddc682157, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/e5d3ebf2f9754d10bd86b39dfbe41bae, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/9212aab04e7f475894c299a12fa72a2a, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/8d7277662164477cbadc589f053d8b0b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/60dbe313e3bf429ca21251e16098ed91, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/c53453a56a1a4f4cbeaee5e8040bacd2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/dafbc51b457249afa2976596a8584201, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/5ef3f887e4bd445383413faf3f591b50, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/6d1044c81b4f430b933d5578be5a53d4, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/ba7b059c0e5845159c6d029b95cf621f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/7222549eae254e61b38d049e15641994, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/a12bb3609f2a4432bbb1e0e39f8c4290, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/06eb14938e004a4ebfcd347abb6b11ad, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/3132e357c12d468b8efe714962927606, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/28ef034d844a44f0853730a2c7a8ced7, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/79e448a645f7405287f9e88b5fca44b9, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/ba52d0eaa3f241eb88f02eadc2142291, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/0fd7d12209334beda3eb33948bab45ec] to archive 2024-11-14T21:34:01,395 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:34:01,431 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/795ecb3d8808428fbb09b6a9d498419d to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/795ecb3d8808428fbb09b6a9d498419d 2024-11-14T21:34:01,433 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/a21ee0eafcae4050a49997f913e1a4e1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/a21ee0eafcae4050a49997f913e1a4e1 2024-11-14T21:34:01,452 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/cf78ff02093e49a383d8505ba7a2b583 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/cf78ff02093e49a383d8505ba7a2b583 2024-11-14T21:34:01,463 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/e4d2d0b18ff64cf18c9621b264549460 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/e4d2d0b18ff64cf18c9621b264549460 2024-11-14T21:34:01,469 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/2fdce2cb2fea4d46803ad70ddc682157 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/2fdce2cb2fea4d46803ad70ddc682157 2024-11-14T21:34:01,484 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/e5d3ebf2f9754d10bd86b39dfbe41bae to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/e5d3ebf2f9754d10bd86b39dfbe41bae 2024-11-14T21:34:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:01,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. as already flushing 2024-11-14T21:34:01,487 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/9212aab04e7f475894c299a12fa72a2a to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/9212aab04e7f475894c299a12fa72a2a 2024-11-14T21:34:01,488 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/8d7277662164477cbadc589f053d8b0b to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/8d7277662164477cbadc589f053d8b0b 2024-11-14T21:34:01,494 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/60dbe313e3bf429ca21251e16098ed91 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/60dbe313e3bf429ca21251e16098ed91 2024-11-14T21:34:01,497 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/c53453a56a1a4f4cbeaee5e8040bacd2 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/c53453a56a1a4f4cbeaee5e8040bacd2 2024-11-14T21:34:01,499 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/dafbc51b457249afa2976596a8584201 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/dafbc51b457249afa2976596a8584201 2024-11-14T21:34:01,500 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/5ef3f887e4bd445383413faf3f591b50 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/5ef3f887e4bd445383413faf3f591b50 2024-11-14T21:34:01,502 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/6d1044c81b4f430b933d5578be5a53d4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/6d1044c81b4f430b933d5578be5a53d4 2024-11-14T21:34:01,503 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/ba7b059c0e5845159c6d029b95cf621f to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/ba7b059c0e5845159c6d029b95cf621f 2024-11-14T21:34:01,504 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/7222549eae254e61b38d049e15641994 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/7222549eae254e61b38d049e15641994 2024-11-14T21:34:01,506 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/a12bb3609f2a4432bbb1e0e39f8c4290 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/a12bb3609f2a4432bbb1e0e39f8c4290 2024-11-14T21:34:01,507 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/06eb14938e004a4ebfcd347abb6b11ad to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/06eb14938e004a4ebfcd347abb6b11ad 2024-11-14T21:34:01,508 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/3132e357c12d468b8efe714962927606 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/3132e357c12d468b8efe714962927606 2024-11-14T21:34:01,510 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/28ef034d844a44f0853730a2c7a8ced7 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/28ef034d844a44f0853730a2c7a8ced7 2024-11-14T21:34:01,511 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/79e448a645f7405287f9e88b5fca44b9 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/79e448a645f7405287f9e88b5fca44b9 2024-11-14T21:34:01,513 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/ba52d0eaa3f241eb88f02eadc2142291 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/ba52d0eaa3f241eb88f02eadc2142291 2024-11-14T21:34:01,514 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/0fd7d12209334beda3eb33948bab45ec to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/0fd7d12209334beda3eb33948bab45ec 2024-11-14T21:34:01,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620101594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:01,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620101624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:01,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:01,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620101628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:01,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620101632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:01,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620101632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:01,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:01,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:01,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:01,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:01,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620101746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620101761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:01,772 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144b3dd97fae2f4b10a271a66c91bb6891_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144b3dd97fae2f4b10a271a66c91bb6891_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:01,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/d8dc45aab5f745ef9a32a8f93bf38d46, store: [table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:34:01,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/d8dc45aab5f745ef9a32a8f93bf38d46 is 175, key is test_row_0/A:col10/1731620040355/Put/seqid=0 2024-11-14T21:34:01,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620101782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:01,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620101783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:01,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620101790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:01,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:01,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:01,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742255_1431 (size=31255) 2024-11-14T21:34:01,828 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=647, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/d8dc45aab5f745ef9a32a8f93bf38d46 2024-11-14T21:34:01,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/2b9175e5b70b406ca9da1b9750a0decc is 50, key is test_row_0/B:col10/1731620040355/Put/seqid=0 2024-11-14T21:34:01,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742256_1432 (size=12301) 2024-11-14T21:34:01,942 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=647 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/2b9175e5b70b406ca9da1b9750a0decc 2024-11-14T21:34:01,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/559064ad58fa401fb406ace92b69b22a is 50, key is test_row_0/C:col10/1731620040355/Put/seqid=0 2024-11-14T21:34:01,997 INFO [Thread-1536 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1536. 2024-11-14T21:34:01,997 DEBUG [Thread-1536 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:34:01,997 DEBUG [Thread-1536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:01,998 DEBUG [Thread-1536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:02,002 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:34:02,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620101977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:02,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:02,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:02,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620101977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:02,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:02,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:02,008 INFO [Thread-1540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1540. 2024-11-14T21:34:02,009 DEBUG [Thread-1540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:34:02,009 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:02,009 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:02,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:34:02,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:02,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:02,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742257_1433 (size=12301) 2024-11-14T21:34:02,017 INFO [Thread-1542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1542. 2024-11-14T21:34:02,017 DEBUG [Thread-1542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:34:02,017 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:02,018 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:02,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:34:02,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620102016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:02,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:02,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:02,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620102017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:02,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:02,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:02,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:02,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:02,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:02,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620102020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:02,029 INFO [Thread-1538 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1538. 2024-11-14T21:34:02,029 DEBUG [Thread-1538 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:34:02,029 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:02,029 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:02,029 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:34:02,044 INFO [Thread-1544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1544. 2024-11-14T21:34:02,044 DEBUG [Thread-1544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:34:02,044 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:02,044 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:02,044 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:34:02,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:02,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1731620102311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:02,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53012 deadline: 1731620102321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:02,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:02,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:02,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:02,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1731620102331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:02,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53028 deadline: 1731620102330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:02,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:02,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:02,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:02,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:02,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:02,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:02,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731620102330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:02,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:02,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:02,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:02,414 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=647 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/559064ad58fa401fb406ace92b69b22a 2024-11-14T21:34:02,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/d8dc45aab5f745ef9a32a8f93bf38d46 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/d8dc45aab5f745ef9a32a8f93bf38d46 2024-11-14T21:34:02,441 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/d8dc45aab5f745ef9a32a8f93bf38d46, entries=150, sequenceid=647, filesize=30.5 K 2024-11-14T21:34:02,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/2b9175e5b70b406ca9da1b9750a0decc as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/2b9175e5b70b406ca9da1b9750a0decc 2024-11-14T21:34:02,449 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/2b9175e5b70b406ca9da1b9750a0decc, entries=150, sequenceid=647, filesize=12.0 K 2024-11-14T21:34:02,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/559064ad58fa401fb406ace92b69b22a as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/559064ad58fa401fb406ace92b69b22a 2024-11-14T21:34:02,469 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/559064ad58fa401fb406ace92b69b22a, entries=150, sequenceid=647, filesize=12.0 K 2024-11-14T21:34:02,471 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~350.95 KB/359376, currentSize=67.09 KB/68700 for c47d7036b7faae0ba1cfffe0348781b1 in 1180ms, sequenceid=647, compaction requested=true 2024-11-14T21:34:02,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:34:02,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:02,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-14T21:34:02,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-14T21:34:02,478 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-14T21:34:02,478 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 5.8090 sec 2024-11-14T21:34:02,480 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 5.8260 sec 2024-11-14T21:34:02,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:02,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c47d7036b7faae0ba1cfffe0348781b1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T21:34:02,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=A 2024-11-14T21:34:02,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:02,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=B 2024-11-14T21:34:02,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:02,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=C 2024-11-14T21:34:02,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:02,826 INFO [Thread-1531 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1531. 2024-11-14T21:34:02,826 DEBUG [Thread-1531 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:34:02,826 DEBUG [Thread-1531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:02,827 DEBUG [Thread-1531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:02,827 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:34:02,843 INFO [Thread-1527 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1527. 2024-11-14T21:34:02,843 DEBUG [Thread-1527 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:34:02,843 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:02,843 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:02,843 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:34:02,844 INFO [Thread-1525 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1525. 2024-11-14T21:34:02,844 DEBUG [Thread-1525 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:34:02,844 DEBUG [Thread-1525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:02,844 DEBUG [Thread-1525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:02,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:34:02,845 INFO [Thread-1533 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1533. 2024-11-14T21:34:02,845 DEBUG [Thread-1533 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:34:02,845 DEBUG [Thread-1533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:02,845 DEBUG [Thread-1533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:02,845 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:34:02,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143f6e4d2bdd394be3a71456ce8a61b1f9_c47d7036b7faae0ba1cfffe0348781b1 is 50, key is test_row_0/A:col10/1731620042821/Put/seqid=0 2024-11-14T21:34:02,852 INFO [Thread-1529 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1529. 2024-11-14T21:34:02,853 DEBUG [Thread-1529 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:34:02,853 DEBUG [Thread-1529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:02,853 DEBUG [Thread-1529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:02,853 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:34:02,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742258_1434 (size=12454) 2024-11-14T21:34:03,261 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:03,263 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143f6e4d2bdd394be3a71456ce8a61b1f9_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143f6e4d2bdd394be3a71456ce8a61b1f9_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:03,264 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/0d035251e29c49768f6f558812e1d056, store: [table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:34:03,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/0d035251e29c49768f6f558812e1d056 is 175, key is test_row_0/A:col10/1731620042821/Put/seqid=0 2024-11-14T21:34:03,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742259_1435 (size=31255) 2024-11-14T21:34:03,669 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=661, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/0d035251e29c49768f6f558812e1d056 2024-11-14T21:34:03,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/0b418eb5994f4845a0095e818fd3b156 is 50, key is test_row_0/B:col10/1731620042821/Put/seqid=0 2024-11-14T21:34:03,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742260_1436 (size=12301) 2024-11-14T21:34:03,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=661 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/0b418eb5994f4845a0095e818fd3b156 2024-11-14T21:34:03,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/5bd7fb5f52cf459581044d3e549ff6cc is 50, key is test_row_0/C:col10/1731620042821/Put/seqid=0 2024-11-14T21:34:03,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742261_1437 (size=12301) 2024-11-14T21:34:03,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=661 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/5bd7fb5f52cf459581044d3e549ff6cc 2024-11-14T21:34:03,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/0d035251e29c49768f6f558812e1d056 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/0d035251e29c49768f6f558812e1d056 2024-11-14T21:34:03,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/0d035251e29c49768f6f558812e1d056, entries=150, sequenceid=661, filesize=30.5 K 2024-11-14T21:34:03,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/0b418eb5994f4845a0095e818fd3b156 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/0b418eb5994f4845a0095e818fd3b156 2024-11-14T21:34:03,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/0b418eb5994f4845a0095e818fd3b156, entries=150, sequenceid=661, filesize=12.0 K 2024-11-14T21:34:03,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/5bd7fb5f52cf459581044d3e549ff6cc as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/5bd7fb5f52cf459581044d3e549ff6cc 2024-11-14T21:34:03,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/5bd7fb5f52cf459581044d3e549ff6cc, entries=150, sequenceid=661, filesize=12.0 K 2024-11-14T21:34:03,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=26.84 KB/27480 for c47d7036b7faae0ba1cfffe0348781b1 in 1067ms, sequenceid=661, compaction requested=true 2024-11-14T21:34:03,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:34:03,893 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:34:03,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:34:03,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:03,894 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:34:03,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:34:03,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:03,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c47d7036b7faae0ba1cfffe0348781b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:34:03,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:03,898 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134464 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:34:03,898 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/A is initiating minor compaction (all files) 2024-11-14T21:34:03,898 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/A in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:03,898 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/ba468eae760c4ddb995952c4594201b8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/1cbf3b6a8c7c4936b9b2f072c8a20f19, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/d8dc45aab5f745ef9a32a8f93bf38d46, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/0d035251e29c49768f6f558812e1d056] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=131.3 K 2024-11-14T21:34:03,898 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:03,898 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. files: [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/ba468eae760c4ddb995952c4594201b8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/1cbf3b6a8c7c4936b9b2f072c8a20f19, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/d8dc45aab5f745ef9a32a8f93bf38d46, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/0d035251e29c49768f6f558812e1d056] 2024-11-14T21:34:03,899 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba468eae760c4ddb995952c4594201b8, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731620038463 2024-11-14T21:34:03,900 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cbf3b6a8c7c4936b9b2f072c8a20f19, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=611, earliestPutTs=1731620040083 2024-11-14T21:34:03,900 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:34:03,900 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/B is initiating minor compaction (all files) 2024-11-14T21:34:03,901 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/B in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:03,901 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/e351c0d37889437593348da04c0fc19f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/ea41a6e6b654428f9cf96362239c78ec, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/2b9175e5b70b406ca9da1b9750a0decc, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/0b418eb5994f4845a0095e818fd3b156] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=48.8 K 2024-11-14T21:34:03,903 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8dc45aab5f745ef9a32a8f93bf38d46, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=647, earliestPutTs=1731620040355 2024-11-14T21:34:03,903 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting e351c0d37889437593348da04c0fc19f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731620038463 2024-11-14T21:34:03,904 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting ea41a6e6b654428f9cf96362239c78ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=611, earliestPutTs=1731620040083 2024-11-14T21:34:03,904 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d035251e29c49768f6f558812e1d056, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=661, earliestPutTs=1731620041612 2024-11-14T21:34:03,909 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b9175e5b70b406ca9da1b9750a0decc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=647, earliestPutTs=1731620040355 2024-11-14T21:34:03,910 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b418eb5994f4845a0095e818fd3b156, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=661, earliestPutTs=1731620041612 2024-11-14T21:34:03,923 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#B#compaction#368 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:03,924 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/c54dd1155eec4ba7ae032fb1df646ab1 is 50, key is test_row_0/B:col10/1731620042821/Put/seqid=0 2024-11-14T21:34:03,926 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:34:03,939 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411149d591c1ed79c427db444a1b4e274a81b_c47d7036b7faae0ba1cfffe0348781b1 store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:34:03,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742262_1438 (size=12439) 2024-11-14T21:34:03,946 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/c54dd1155eec4ba7ae032fb1df646ab1 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/c54dd1155eec4ba7ae032fb1df646ab1 2024-11-14T21:34:03,961 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/B of c47d7036b7faae0ba1cfffe0348781b1 into c54dd1155eec4ba7ae032fb1df646ab1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:03,961 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:34:03,961 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/B, priority=12, startTime=1731620043894; duration=0sec 2024-11-14T21:34:03,961 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:03,961 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:B 2024-11-14T21:34:03,961 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:34:03,963 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:34:03,963 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): c47d7036b7faae0ba1cfffe0348781b1/C is initiating minor compaction (all files) 2024-11-14T21:34:03,963 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c47d7036b7faae0ba1cfffe0348781b1/C in TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:03,963 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/16de88725b674d2ca719dc770a91e6b4, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/96928cffe2c14a82a305a1c1441c3795, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/559064ad58fa401fb406ace92b69b22a, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/5bd7fb5f52cf459581044d3e549ff6cc] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp, totalSize=48.8 K 2024-11-14T21:34:03,963 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 16de88725b674d2ca719dc770a91e6b4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731620038463 2024-11-14T21:34:03,964 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 96928cffe2c14a82a305a1c1441c3795, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=611, earliestPutTs=1731620040083 2024-11-14T21:34:03,964 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 559064ad58fa401fb406ace92b69b22a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=647, earliestPutTs=1731620040355 2024-11-14T21:34:03,964 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bd7fb5f52cf459581044d3e549ff6cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=661, earliestPutTs=1731620041612 2024-11-14T21:34:03,982 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411149d591c1ed79c427db444a1b4e274a81b_c47d7036b7faae0ba1cfffe0348781b1, store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:34:03,982 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149d591c1ed79c427db444a1b4e274a81b_c47d7036b7faae0ba1cfffe0348781b1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:34:03,986 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#C#compaction#370 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:03,986 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/21888b8bf6184898aeac64231d313a35 is 50, key is test_row_0/C:col10/1731620042821/Put/seqid=0 2024-11-14T21:34:04,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742264_1440 (size=12439) 2024-11-14T21:34:04,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742263_1439 (size=4469) 2024-11-14T21:34:04,436 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c47d7036b7faae0ba1cfffe0348781b1#A#compaction#369 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:04,437 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/d33391545adf43bd94f24b5bba272817 is 175, key is test_row_0/A:col10/1731620042821/Put/seqid=0 2024-11-14T21:34:04,459 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/21888b8bf6184898aeac64231d313a35 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/21888b8bf6184898aeac64231d313a35 2024-11-14T21:34:04,484 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/C of c47d7036b7faae0ba1cfffe0348781b1 into 21888b8bf6184898aeac64231d313a35(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:04,484 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:34:04,484 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/C, priority=12, startTime=1731620043894; duration=0sec 2024-11-14T21:34:04,484 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:04,484 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:C 2024-11-14T21:34:04,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742265_1441 (size=31393) 2024-11-14T21:34:04,562 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/d33391545adf43bd94f24b5bba272817 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/d33391545adf43bd94f24b5bba272817 2024-11-14T21:34:04,581 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c47d7036b7faae0ba1cfffe0348781b1/A of c47d7036b7faae0ba1cfffe0348781b1 into d33391545adf43bd94f24b5bba272817(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:04,581 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c47d7036b7faae0ba1cfffe0348781b1: 2024-11-14T21:34:04,582 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., storeName=c47d7036b7faae0ba1cfffe0348781b1/A, priority=12, startTime=1731620043893; duration=0sec 2024-11-14T21:34:04,582 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:04,582 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c47d7036b7faae0ba1cfffe0348781b1:A 2024-11-14T21:34:04,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-14T21:34:04,863 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:34:04,863 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T21:34:04,863 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-11-14T21:34:04,863 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-11-14T21:34:04,863 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 143 2024-11-14T21:34:04,863 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 115 2024-11-14T21:34:04,863 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-14T21:34:04,863 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T21:34:04,863 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T21:34:04,863 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 630 2024-11-14T21:34:04,863 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1890 rows 2024-11-14T21:34:04,863 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 628 2024-11-14T21:34:04,863 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1884 rows 2024-11-14T21:34:04,863 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 624 2024-11-14T21:34:04,863 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1871 rows 2024-11-14T21:34:04,863 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 635 2024-11-14T21:34:04,863 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1902 rows 2024-11-14T21:34:04,863 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 627 2024-11-14T21:34:04,863 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1881 rows 2024-11-14T21:34:04,863 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T21:34:04,864 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T21:34:04,864 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:04,864 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:04,866 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:34:04,866 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T21:34:04,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T21:34:04,874 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731620044873"}]},"ts":"1731620044873"} 2024-11-14T21:34:04,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T21:34:04,876 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T21:34:04,876 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T21:34:04,876 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T21:34:04,879 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c47d7036b7faae0ba1cfffe0348781b1, UNASSIGN}] 2024-11-14T21:34:04,880 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=95, ppid=94, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c47d7036b7faae0ba1cfffe0348781b1, UNASSIGN 2024-11-14T21:34:04,882 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=95 updating hbase:meta row=c47d7036b7faae0ba1cfffe0348781b1, regionState=CLOSING, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:04,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c47d7036b7faae0ba1cfffe0348781b1, UNASSIGN because future has completed 2024-11-14T21:34:04,884 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T21:34:04,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; CloseRegionProcedure c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560}] 2024-11-14T21:34:04,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T21:34:05,037 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] handler.UnassignRegionHandler(122): Close c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:05,037 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T21:34:05,037 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(1722): Closing c47d7036b7faae0ba1cfffe0348781b1, disabling compactions & flushes 2024-11-14T21:34:05,037 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:05,037 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:05,037 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. after waiting 0 ms 2024-11-14T21:34:05,037 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:05,037 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(2902): Flushing c47d7036b7faae0ba1cfffe0348781b1 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-14T21:34:05,038 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=A 2024-11-14T21:34:05,038 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:05,038 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=B 2024-11-14T21:34:05,038 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:05,038 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c47d7036b7faae0ba1cfffe0348781b1, store=C 2024-11-14T21:34:05,038 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:05,049 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114293893ef1d3d4aee8d7e59d04ee18a21_c47d7036b7faae0ba1cfffe0348781b1 is 50, key is test_row_0/A:col10/1731620042843/Put/seqid=0 2024-11-14T21:34:05,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742266_1442 (size=12454) 2024-11-14T21:34:05,066 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:05,082 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114293893ef1d3d4aee8d7e59d04ee18a21_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114293893ef1d3d4aee8d7e59d04ee18a21_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:05,085 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/cd1471bac7fb44c5b5ba89fca1456c87, store: [table=TestAcidGuarantees family=A region=c47d7036b7faae0ba1cfffe0348781b1] 2024-11-14T21:34:05,086 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/cd1471bac7fb44c5b5ba89fca1456c87 is 175, key is test_row_0/A:col10/1731620042843/Put/seqid=0 2024-11-14T21:34:05,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742267_1443 (size=31255) 2024-11-14T21:34:05,146 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=671, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/cd1471bac7fb44c5b5ba89fca1456c87 2024-11-14T21:34:05,192 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/9078aa3a81b34fdaad766a90c23fe458 is 50, key is test_row_0/B:col10/1731620042843/Put/seqid=0 2024-11-14T21:34:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T21:34:05,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742268_1444 (size=12301) 2024-11-14T21:34:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T21:34:05,622 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=671 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/9078aa3a81b34fdaad766a90c23fe458 2024-11-14T21:34:05,641 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/3eb7b39574b848ddb39a727f43f85d26 is 50, key is test_row_0/C:col10/1731620042843/Put/seqid=0 2024-11-14T21:34:05,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742269_1445 (size=12301) 2024-11-14T21:34:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T21:34:06,064 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=671 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/3eb7b39574b848ddb39a727f43f85d26 2024-11-14T21:34:06,073 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/A/cd1471bac7fb44c5b5ba89fca1456c87 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/cd1471bac7fb44c5b5ba89fca1456c87 2024-11-14T21:34:06,083 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/cd1471bac7fb44c5b5ba89fca1456c87, entries=150, sequenceid=671, filesize=30.5 K 2024-11-14T21:34:06,084 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/B/9078aa3a81b34fdaad766a90c23fe458 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/9078aa3a81b34fdaad766a90c23fe458 2024-11-14T21:34:06,096 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/9078aa3a81b34fdaad766a90c23fe458, entries=150, sequenceid=671, filesize=12.0 K 2024-11-14T21:34:06,106 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/.tmp/C/3eb7b39574b848ddb39a727f43f85d26 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/3eb7b39574b848ddb39a727f43f85d26 2024-11-14T21:34:06,116 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/3eb7b39574b848ddb39a727f43f85d26, entries=150, sequenceid=671, filesize=12.0 K 2024-11-14T21:34:06,117 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c47d7036b7faae0ba1cfffe0348781b1 in 1080ms, sequenceid=671, compaction requested=false 2024-11-14T21:34:06,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/ba468eae760c4ddb995952c4594201b8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/1cbf3b6a8c7c4936b9b2f072c8a20f19, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/d8dc45aab5f745ef9a32a8f93bf38d46, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/0d035251e29c49768f6f558812e1d056] to archive 2024-11-14T21:34:06,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:34:06,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/ba468eae760c4ddb995952c4594201b8 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/ba468eae760c4ddb995952c4594201b8 2024-11-14T21:34:06,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/1cbf3b6a8c7c4936b9b2f072c8a20f19 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/1cbf3b6a8c7c4936b9b2f072c8a20f19 2024-11-14T21:34:06,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/d8dc45aab5f745ef9a32a8f93bf38d46 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/d8dc45aab5f745ef9a32a8f93bf38d46 2024-11-14T21:34:06,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/0d035251e29c49768f6f558812e1d056 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/0d035251e29c49768f6f558812e1d056 2024-11-14T21:34:06,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/e351c0d37889437593348da04c0fc19f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/ea41a6e6b654428f9cf96362239c78ec, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/2b9175e5b70b406ca9da1b9750a0decc, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/0b418eb5994f4845a0095e818fd3b156] to archive 2024-11-14T21:34:06,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:34:06,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/e351c0d37889437593348da04c0fc19f to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/e351c0d37889437593348da04c0fc19f 2024-11-14T21:34:06,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/ea41a6e6b654428f9cf96362239c78ec to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/ea41a6e6b654428f9cf96362239c78ec 2024-11-14T21:34:06,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/2b9175e5b70b406ca9da1b9750a0decc to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/2b9175e5b70b406ca9da1b9750a0decc 2024-11-14T21:34:06,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/0b418eb5994f4845a0095e818fd3b156 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/0b418eb5994f4845a0095e818fd3b156 2024-11-14T21:34:06,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/16de88725b674d2ca719dc770a91e6b4, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/96928cffe2c14a82a305a1c1441c3795, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/559064ad58fa401fb406ace92b69b22a, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/5bd7fb5f52cf459581044d3e549ff6cc] to archive 2024-11-14T21:34:06,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:34:06,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/16de88725b674d2ca719dc770a91e6b4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/16de88725b674d2ca719dc770a91e6b4 2024-11-14T21:34:06,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/96928cffe2c14a82a305a1c1441c3795 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/96928cffe2c14a82a305a1c1441c3795 2024-11-14T21:34:06,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/559064ad58fa401fb406ace92b69b22a to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/559064ad58fa401fb406ace92b69b22a 2024-11-14T21:34:06,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/5bd7fb5f52cf459581044d3e549ff6cc to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/5bd7fb5f52cf459581044d3e549ff6cc 2024-11-14T21:34:06,213 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/recovered.edits/674.seqid, newMaxSeqId=674, maxSeqId=4 2024-11-14T21:34:06,214 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. 2024-11-14T21:34:06,214 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(1676): Region close journal for c47d7036b7faae0ba1cfffe0348781b1: Waiting for close lock at 1731620045037Running coprocessor pre-close hooks at 1731620045037Disabling compacts and flushes for region at 1731620045037Disabling writes for close at 1731620045037Obtaining lock to block concurrent updates at 1731620045037Preparing flush snapshotting stores in c47d7036b7faae0ba1cfffe0348781b1 at 1731620045037Finished memstore snapshotting TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731620045038 (+1 ms)Flushing stores of TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1. at 1731620045039 (+1 ms)Flushing c47d7036b7faae0ba1cfffe0348781b1/A: creating writer at 1731620045039Flushing mob file c47d7036b7faae0ba1cfffe0348781b1/A: appending metadata at 1731620045049 (+10 ms)Flushing mob file c47d7036b7faae0ba1cfffe0348781b1/A: closing flushed file at 1731620045049Flushing c47d7036b7faae0ba1cfffe0348781b1/A: appending metadata at 1731620045085 (+36 ms)Flushing c47d7036b7faae0ba1cfffe0348781b1/A: closing flushed file at 1731620045085Flushing c47d7036b7faae0ba1cfffe0348781b1/B: creating writer at 1731620045185 (+100 ms)Flushing c47d7036b7faae0ba1cfffe0348781b1/B: appending metadata at 1731620045192 (+7 ms)Flushing c47d7036b7faae0ba1cfffe0348781b1/B: closing flushed file at 1731620045192Flushing c47d7036b7faae0ba1cfffe0348781b1/C: creating writer at 1731620045635 (+443 ms)Flushing c47d7036b7faae0ba1cfffe0348781b1/C: appending metadata at 1731620045640 (+5 ms)Flushing c47d7036b7faae0ba1cfffe0348781b1/C: closing flushed file at 1731620045640Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1dd466a3: reopening flushed file at 1731620046068 (+428 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@674e796b: reopening flushed file at 1731620046084 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f0f5064: reopening flushed file at 1731620046096 (+12 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c47d7036b7faae0ba1cfffe0348781b1 in 1080ms, sequenceid=671, compaction requested=false at 1731620046117 (+21 ms)Writing region close event to WAL at 1731620046200 (+83 ms)Running coprocessor post-close hooks at 1731620046214 (+14 ms)Closed at 1731620046214 2024-11-14T21:34:06,217 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] handler.UnassignRegionHandler(157): Closed c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:06,217 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=95 updating hbase:meta row=c47d7036b7faae0ba1cfffe0348781b1, regionState=CLOSED 2024-11-14T21:34:06,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE, hasLock=false; CloseRegionProcedure c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 because future has completed 2024-11-14T21:34:06,222 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-14T21:34:06,222 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; CloseRegionProcedure c47d7036b7faae0ba1cfffe0348781b1, server=ef788a4cade8,37683,1731619918560 in 1.3360 sec 2024-11-14T21:34:06,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-14T21:34:06,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c47d7036b7faae0ba1cfffe0348781b1, UNASSIGN in 1.3430 sec 2024-11-14T21:34:06,226 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-14T21:34:06,226 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.3490 sec 2024-11-14T21:34:06,231 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731620046231"}]},"ts":"1731620046231"} 2024-11-14T21:34:06,234 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T21:34:06,234 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T21:34:06,236 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3680 sec 2024-11-14T21:34:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T21:34:07,023 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:34:07,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T21:34:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:34:07,031 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=97, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:34:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T21:34:07,035 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=97, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:34:07,078 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:07,085 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A, FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B, FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C, FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/recovered.edits] 2024-11-14T21:34:07,098 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/cd1471bac7fb44c5b5ba89fca1456c87 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/cd1471bac7fb44c5b5ba89fca1456c87 2024-11-14T21:34:07,110 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/d33391545adf43bd94f24b5bba272817 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/A/d33391545adf43bd94f24b5bba272817 2024-11-14T21:34:07,123 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/9078aa3a81b34fdaad766a90c23fe458 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/9078aa3a81b34fdaad766a90c23fe458 2024-11-14T21:34:07,135 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/c54dd1155eec4ba7ae032fb1df646ab1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/B/c54dd1155eec4ba7ae032fb1df646ab1 2024-11-14T21:34:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T21:34:07,141 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/21888b8bf6184898aeac64231d313a35 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/21888b8bf6184898aeac64231d313a35 2024-11-14T21:34:07,146 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/3eb7b39574b848ddb39a727f43f85d26 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/C/3eb7b39574b848ddb39a727f43f85d26 2024-11-14T21:34:07,155 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/recovered.edits/674.seqid to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1/recovered.edits/674.seqid 2024-11-14T21:34:07,156 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:07,156 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T21:34:07,157 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T21:34:07,158 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T21:34:07,167 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114071585cac6444beb8f7724b824c10d6e_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114071585cac6444beb8f7724b824c10d6e_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:07,171 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111411f21a45674e49b29989e42387fa1916_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111411f21a45674e49b29989e42387fa1916_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:07,178 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114133a7830917a4956a4555c90a2bdef4a_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114133a7830917a4956a4555c90a2bdef4a_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:07,182 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141c50ee358b134fd6a623c304dd690092_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141c50ee358b134fd6a623c304dd690092_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:07,183 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114293893ef1d3d4aee8d7e59d04ee18a21_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114293893ef1d3d4aee8d7e59d04ee18a21_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:07,185 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143f6e4d2bdd394be3a71456ce8a61b1f9_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143f6e4d2bdd394be3a71456ce8a61b1f9_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:07,186 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144b3dd97fae2f4b10a271a66c91bb6891_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144b3dd97fae2f4b10a271a66c91bb6891_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:07,187 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114565e621e63da48aea09eca75c5510cd9_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114565e621e63da48aea09eca75c5510cd9_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:07,189 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145ac9e520c2e8431c8f3755948907b3ed_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145ac9e520c2e8431c8f3755948907b3ed_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:07,190 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145b12507f8b154d998022f90dff494a59_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145b12507f8b154d998022f90dff494a59_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:07,193 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114730f99cb35fb4da1bfe34c30ceab5a9d_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114730f99cb35fb4da1bfe34c30ceab5a9d_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:07,196 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148226e5cc03174ed1af84943bf15f6a17_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148226e5cc03174ed1af84943bf15f6a17_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:07,203 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111486a8fe1e1846499a93cba06563fc6c40_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111486a8fe1e1846499a93cba06563fc6c40_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:07,204 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a67cd46904d0490895405d0b89211be4_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a67cd46904d0490895405d0b89211be4_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:07,205 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b181de63c7f14ec698ed454c2f2c0de4_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b181de63c7f14ec698ed454c2f2c0de4_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:07,209 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bafbfaf515a641c386a697e275cec94e_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bafbfaf515a641c386a697e275cec94e_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:07,210 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114db9e6b157780495894213978628df539_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114db9e6b157780495894213978628df539_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:07,216 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e93f6be5ef5a4b0da5bc78c6269f464d_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e93f6be5ef5a4b0da5bc78c6269f464d_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:07,217 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ec26c149e9bf4afaaf13ac4a1eec3e1b_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ec26c149e9bf4afaaf13ac4a1eec3e1b_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:07,230 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fc5f3ec123794c5c891f1a92f8e74609_c47d7036b7faae0ba1cfffe0348781b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fc5f3ec123794c5c891f1a92f8e74609_c47d7036b7faae0ba1cfffe0348781b1 2024-11-14T21:34:07,232 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T21:34:07,240 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=97, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:34:07,243 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T21:34:07,246 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T21:34:07,247 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=97, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:34:07,247 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T21:34:07,248 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731620047247"}]},"ts":"9223372036854775807"} 2024-11-14T21:34:07,249 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T21:34:07,249 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c47d7036b7faae0ba1cfffe0348781b1, NAME => 'TestAcidGuarantees,,1731620019600.c47d7036b7faae0ba1cfffe0348781b1.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T21:34:07,249 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T21:34:07,250 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731620047250"}]},"ts":"9223372036854775807"} 2024-11-14T21:34:07,255 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T21:34:07,256 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=97, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:34:07,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 232 msec 2024-11-14T21:34:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T21:34:07,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T21:34:07,351 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:34:07,363 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 246) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1436642103_22 at /127.0.0.1:56832 [Waiting for operation #169] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1436642103_22 at /127.0.0.1:56816 [Waiting for operation #155] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_967508010_22 at /127.0.0.1:56830 [Waiting for operation #164] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_967508010_22 at /127.0.0.1:60488 [Waiting for operation #226] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1167 (was 957) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1403 (was 1230) - AvailableMemoryMB LEAK? - 2024-11-14T21:34:07,374 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=1167, ProcessCount=11, AvailableMemoryMB=1402 2024-11-14T21:34:07,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T21:34:07,376 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T21:34:07,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=98, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T21:34:07,379 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=98, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T21:34:07,379 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:07,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 98 2024-11-14T21:34:07,381 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=98, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T21:34:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-14T21:34:07,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742270_1446 (size=1061) 2024-11-14T21:34:07,395 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275 2024-11-14T21:34:07,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742271_1447 (size=53) 2024-11-14T21:34:07,419 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:34:07,419 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 81ffab6ef2cdc120372df3297622f922, disabling compactions & flushes 2024-11-14T21:34:07,419 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:07,419 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:07,419 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. after waiting 0 ms 2024-11-14T21:34:07,419 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:07,419 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:07,419 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 81ffab6ef2cdc120372df3297622f922: Waiting for close lock at 1731620047419Disabling compacts and flushes for region at 1731620047419Disabling writes for close at 1731620047419Writing region close event to WAL at 1731620047419Closed at 1731620047419 2024-11-14T21:34:07,421 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=98, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T21:34:07,421 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731620047421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731620047421"}]},"ts":"1731620047421"} 2024-11-14T21:34:07,423 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T21:34:07,424 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=98, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T21:34:07,424 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731620047424"}]},"ts":"1731620047424"} 2024-11-14T21:34:07,425 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T21:34:07,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81ffab6ef2cdc120372df3297622f922, ASSIGN}] 2024-11-14T21:34:07,427 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=99, ppid=98, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81ffab6ef2cdc120372df3297622f922, ASSIGN 2024-11-14T21:34:07,428 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=99, ppid=98, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81ffab6ef2cdc120372df3297622f922, ASSIGN; state=OFFLINE, location=ef788a4cade8,37683,1731619918560; forceNewPlan=false, retain=false 2024-11-14T21:34:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-14T21:34:07,583 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=99 updating hbase:meta row=81ffab6ef2cdc120372df3297622f922, regionState=OPENING, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:07,586 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81ffab6ef2cdc120372df3297622f922, ASSIGN because future has completed 2024-11-14T21:34:07,587 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560}] 2024-11-14T21:34:07,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-14T21:34:07,745 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:07,746 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7752): Opening region: {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:34:07,746 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:07,747 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:34:07,747 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7794): checking encryption for 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:07,747 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7797): checking classloading for 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:07,754 INFO [StoreOpener-81ffab6ef2cdc120372df3297622f922-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:07,767 INFO [StoreOpener-81ffab6ef2cdc120372df3297622f922-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:34:07,768 INFO [StoreOpener-81ffab6ef2cdc120372df3297622f922-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81ffab6ef2cdc120372df3297622f922 columnFamilyName A 2024-11-14T21:34:07,768 DEBUG [StoreOpener-81ffab6ef2cdc120372df3297622f922-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:07,768 INFO [StoreOpener-81ffab6ef2cdc120372df3297622f922-1 {}] regionserver.HStore(327): Store=81ffab6ef2cdc120372df3297622f922/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:34:07,768 INFO [StoreOpener-81ffab6ef2cdc120372df3297622f922-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:07,769 INFO [StoreOpener-81ffab6ef2cdc120372df3297622f922-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:34:07,770 INFO [StoreOpener-81ffab6ef2cdc120372df3297622f922-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81ffab6ef2cdc120372df3297622f922 columnFamilyName B 2024-11-14T21:34:07,770 DEBUG [StoreOpener-81ffab6ef2cdc120372df3297622f922-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:07,770 INFO [StoreOpener-81ffab6ef2cdc120372df3297622f922-1 {}] regionserver.HStore(327): Store=81ffab6ef2cdc120372df3297622f922/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:34:07,770 INFO [StoreOpener-81ffab6ef2cdc120372df3297622f922-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:07,771 INFO [StoreOpener-81ffab6ef2cdc120372df3297622f922-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:34:07,772 INFO [StoreOpener-81ffab6ef2cdc120372df3297622f922-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81ffab6ef2cdc120372df3297622f922 columnFamilyName C 2024-11-14T21:34:07,772 DEBUG [StoreOpener-81ffab6ef2cdc120372df3297622f922-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:07,772 INFO [StoreOpener-81ffab6ef2cdc120372df3297622f922-1 {}] regionserver.HStore(327): Store=81ffab6ef2cdc120372df3297622f922/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:34:07,772 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:07,772 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1038): replaying wal for 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:07,773 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:07,773 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:07,775 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1048): stopping wal replay for 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:07,775 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1060): Cleaning up temporary data for 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:07,776 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T21:34:07,782 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1093): writing seq id for 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:07,795 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T21:34:07,796 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1114): Opened 81ffab6ef2cdc120372df3297622f922; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67999222, jitterRate=0.013267368078231812}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T21:34:07,796 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:07,797 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1006): Region open journal for 81ffab6ef2cdc120372df3297622f922: Running coprocessor pre-open hook at 1731620047747Writing region info on filesystem at 1731620047747Initializing all the Stores at 1731620047748 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731620047748Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731620047750 (+2 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731620047750Cleaning up temporary data from old regions at 1731620047775 (+25 ms)Running coprocessor post-open hooks at 1731620047796 (+21 ms)Region opened successfully at 1731620047797 (+1 ms) 2024-11-14T21:34:07,798 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., pid=100, masterSystemTime=1731620047738 2024-11-14T21:34:07,800 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:07,800 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:07,801 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=99 updating hbase:meta row=81ffab6ef2cdc120372df3297622f922, regionState=OPEN, openSeqNum=2, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:07,810 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 because future has completed 2024-11-14T21:34:07,817 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-14T21:34:07,817 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; OpenRegionProcedure 81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 in 229 msec 2024-11-14T21:34:07,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-14T21:34:07,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81ffab6ef2cdc120372df3297622f922, ASSIGN in 393 msec 2024-11-14T21:34:07,822 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=98, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T21:34:07,823 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731620047822"}]},"ts":"1731620047822"} 2024-11-14T21:34:07,825 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T21:34:07,825 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=98, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T21:34:07,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 449 msec 2024-11-14T21:34:08,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-14T21:34:08,025 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:34:08,026 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4626c9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,026 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:34:08,026 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:34:08,028 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:34:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:34:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:34:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5da8415c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:34:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:34:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:08,033 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:34:08,034 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f9dd1d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,034 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:34:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:34:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:34:08,037 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:34:08,038 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T21:34:08,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is ef788a4cade8,38255,1731619917503 2024-11-14T21:34:08,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1fa70247 2024-11-14T21:34:08,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T21:34:08,040 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T21:34:08,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20502da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,043 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:34:08,043 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:34:08,047 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:34:08,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:34:08,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:34:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66e8fcd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:34:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:34:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:08,051 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:34:08,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d9c2563, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,056 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1decc1d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,056 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:34:08,056 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:34:08,057 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:34:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:34:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:34:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59fa1a61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:34:08,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:34:08,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:08,064 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:34:08,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73ce76c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2700619f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,069 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:34:08,069 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:34:08,069 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:34:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:34:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:34:08,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50bd95dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:34:08,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:34:08,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:08,071 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:34:08,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b7e369, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7342f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,072 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:34:08,072 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:34:08,072 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:34:08,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:34:08,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:34:08,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c2c6a4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:34:08,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:34:08,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:08,074 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:34:08,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11b6f585, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c8b6930, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,075 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:34:08,075 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:34:08,076 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:34:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:34:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:34:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@791ea18c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:34:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:34:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:08,077 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:34:08,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c7c55ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,078 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130a1137, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,078 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:34:08,078 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:34:08,079 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:34:08,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:34:08,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:34:08,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cd38125, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:34:08,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:34:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:08,080 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:34:08,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@299b8ec3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d25eec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,082 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:34:08,082 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:34:08,083 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:34:08,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:34:08,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:34:08,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1706828f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:34:08,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:34:08,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:08,084 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:34:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f4cdeda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ecc4d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,094 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:34:08,094 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:34:08,095 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:34:08,095 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:34:08,095 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:34:08,096 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34ef8859, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,096 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:34:08,096 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:34:08,096 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:08,098 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:34:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1313c7b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ec80f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,102 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:34:08,103 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:34:08,106 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:34:08,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:34:08,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:34:08,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@febdf61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:34:08,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:34:08,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:08,108 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:34:08,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb36252, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@617b85c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,110 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:34:08,111 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:34:08,111 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:34:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:34:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:34:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6baf19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:34:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:34:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:08,113 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:34:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@201d0b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:08,119 DEBUG [Thread-2026 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:34:08,119 DEBUG [Thread-2011 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:34:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:34:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:34:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:34:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:34:08,123 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:34:08,123 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:34:08,124 DEBUG [Thread-2019 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:34:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:34:08,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:34:08,127 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:34:08,127 DEBUG [Thread-2030 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:34:08,127 DEBUG [Thread-2017 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:34:08,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:34:08,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:34:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:34:08,131 DEBUG [Thread-2028 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:34:08,132 DEBUG [Thread-2028 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:34:08,133 DEBUG [Thread-2028 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:34:08,135 DEBUG [Thread-2022 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:34:08,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:34:08,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:34:08,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:34:08,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:34:08,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:34:08,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:34:08,137 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:34:08,138 DEBUG [Thread-2013 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:34:08,138 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:34:08,138 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:34:08,138 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:34:08,140 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:34:08,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:34:08,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:34:08,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:34:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:34:08,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:34:08,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:34:08,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-14T21:34:08,145 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:34:08,145 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:34:08,147 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:34:08,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:34:08,147 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:34:08,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T21:34:08,151 DEBUG [Thread-2024 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:34:08,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:34:08,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:34:08,153 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:34:08,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:34:08,163 DEBUG [Thread-2015 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:34:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:34:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:34:08,175 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:34:08,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2] 2024-11-14T21:34:08,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:08,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T21:34:08,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:08,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:08,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:08,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:08,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:08,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:08,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T21:34:08,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/57dd4b418f42434d90bb4af6b642f938 is 50, key is test_row_0/A:col10/1731620048211/Put/seqid=0 2024-11-14T21:34:08,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742272_1448 (size=11997) 2024-11-14T21:34:08,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T21:34:08,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:08,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:08,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:08,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:08,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:08,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:08,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620108285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:08,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620108286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:08,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620108288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:08,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620108292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:08,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731620048284, max timestamp=1731620048300Num uniques 859; 2024-11-14T21:34:08,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:34:08,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731620048284, max timestamp=1731620048300Num uniques 859; 2024-11-14T21:34:08,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:34:08,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731620048284, max timestamp=1731620048300Num uniques 859; 2024-11-14T21:34:08,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:34:08,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620108423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:08,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:08,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:08,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620108425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:08,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:08,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:08,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620108427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:08,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:08,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620108427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:08,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:08,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:08,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620108429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:08,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:08,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T21:34:08,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:08,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:08,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:08,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:08,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:08,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T21:34:08,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:08,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620108540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:08,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:08,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620108550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:08,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620108551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:08,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620108551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:08,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:08,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620108551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:08,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:08,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:08,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:08,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T21:34:08,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:08,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:08,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:08,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:08,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:08,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:08,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/57dd4b418f42434d90bb4af6b642f938 2024-11-14T21:34:08,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/17f68d20f8214dee9ae8a9cc6ed0a5f6 is 50, key is test_row_0/B:col10/1731620048211/Put/seqid=0 2024-11-14T21:34:08,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742273_1449 (size=9657) 2024-11-14T21:34:08,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/17f68d20f8214dee9ae8a9cc6ed0a5f6 2024-11-14T21:34:08,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:08,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620108760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:08,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:08,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T21:34:08,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:08,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:08,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:08,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:08,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:08,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:08,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:08,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620108771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:08,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:08,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:08,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620108771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:08,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:08,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T21:34:08,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:08,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620108783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:08,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:08,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:08,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620108783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:08,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:08,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:08,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/fd7c5a8794da4daf9d4852e9c83ad136 is 50, key is test_row_0/C:col10/1731620048211/Put/seqid=0 2024-11-14T21:34:08,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742274_1450 (size=9657) 2024-11-14T21:34:08,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/fd7c5a8794da4daf9d4852e9c83ad136 2024-11-14T21:34:08,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/57dd4b418f42434d90bb4af6b642f938 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/57dd4b418f42434d90bb4af6b642f938 2024-11-14T21:34:08,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/57dd4b418f42434d90bb4af6b642f938, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T21:34:08,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/17f68d20f8214dee9ae8a9cc6ed0a5f6 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/17f68d20f8214dee9ae8a9cc6ed0a5f6 2024-11-14T21:34:08,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/17f68d20f8214dee9ae8a9cc6ed0a5f6, entries=100, sequenceid=12, filesize=9.4 K 2024-11-14T21:34:08,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/fd7c5a8794da4daf9d4852e9c83ad136 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/fd7c5a8794da4daf9d4852e9c83ad136 2024-11-14T21:34:08,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/fd7c5a8794da4daf9d4852e9c83ad136, entries=100, sequenceid=12, filesize=9.4 K 2024-11-14T21:34:08,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 81ffab6ef2cdc120372df3297622f922 in 704ms, sequenceid=12, compaction requested=false 2024-11-14T21:34:08,917 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T21:34:08,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:08,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T21:34:08,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:08,928 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-14T21:34:08,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:08,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:08,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:08,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:08,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:08,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:08,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/cf48cf0160f447b3a26822591859c13c is 50, key is test_row_0/A:col10/1731620048424/Put/seqid=0 2024-11-14T21:34:08,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742275_1451 (size=12001) 2024-11-14T21:34:08,992 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/cf48cf0160f447b3a26822591859c13c 2024-11-14T21:34:09,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/30ce30323cd44fa3a093982ee44bb095 is 50, key is test_row_0/B:col10/1731620048424/Put/seqid=0 2024-11-14T21:34:09,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742276_1452 (size=12001) 2024-11-14T21:34:09,074 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/30ce30323cd44fa3a093982ee44bb095 2024-11-14T21:34:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:09,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:09,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/db3368a49b914caa9b2096f5e33e2363 is 50, key is test_row_0/C:col10/1731620048424/Put/seqid=0 2024-11-14T21:34:09,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742277_1453 (size=12001) 2024-11-14T21:34:09,119 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/db3368a49b914caa9b2096f5e33e2363 2024-11-14T21:34:09,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620109110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620109129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/cf48cf0160f447b3a26822591859c13c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/cf48cf0160f447b3a26822591859c13c 2024-11-14T21:34:09,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620109133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620109133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620109133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,163 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/cf48cf0160f447b3a26822591859c13c, entries=150, sequenceid=48, filesize=11.7 K 2024-11-14T21:34:09,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/30ce30323cd44fa3a093982ee44bb095 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/30ce30323cd44fa3a093982ee44bb095 2024-11-14T21:34:09,200 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/30ce30323cd44fa3a093982ee44bb095, entries=150, sequenceid=48, filesize=11.7 K 2024-11-14T21:34:09,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/db3368a49b914caa9b2096f5e33e2363 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/db3368a49b914caa9b2096f5e33e2363 2024-11-14T21:34:09,231 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/db3368a49b914caa9b2096f5e33e2363, entries=150, sequenceid=48, filesize=11.7 K 2024-11-14T21:34:09,233 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=60.38 KB/61830 for 81ffab6ef2cdc120372df3297622f922 in 305ms, sequenceid=48, compaction requested=false 2024-11-14T21:34:09,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:09,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:09,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-14T21:34:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-14T21:34:09,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T21:34:09,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:09,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:09,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:09,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:09,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:09,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:09,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-14T21:34:09,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0930 sec 2024-11-14T21:34:09,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.1160 sec 2024-11-14T21:34:09,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/af210329842c4032983843b9353ecb9d is 50, key is test_row_0/A:col10/1731620049106/Put/seqid=0 2024-11-14T21:34:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T21:34:09,292 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:34:09,300 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:34:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-14T21:34:09,302 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:34:09,302 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:34:09,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:34:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T21:34:09,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742278_1454 (size=14341) 2024-11-14T21:34:09,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/af210329842c4032983843b9353ecb9d 2024-11-14T21:34:09,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/4133e34e9c994a50801fdd2097d6dfe3 is 50, key is test_row_0/B:col10/1731620049106/Put/seqid=0 2024-11-14T21:34:09,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620109324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620109341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742279_1455 (size=12001) 2024-11-14T21:34:09,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620109343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620109348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620109351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/4133e34e9c994a50801fdd2097d6dfe3 2024-11-14T21:34:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T21:34:09,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/adc128f28a4d4cf79cb7671d14d58477 is 50, key is test_row_0/C:col10/1731620049106/Put/seqid=0 2024-11-14T21:34:09,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T21:34:09,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:09,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:09,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:09,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:09,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:09,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742280_1456 (size=12001) 2024-11-14T21:34:09,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/adc128f28a4d4cf79cb7671d14d58477 2024-11-14T21:34:09,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/af210329842c4032983843b9353ecb9d as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/af210329842c4032983843b9353ecb9d 2024-11-14T21:34:09,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620109463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620109471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620109471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620109480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620109481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/af210329842c4032983843b9353ecb9d, entries=200, sequenceid=61, filesize=14.0 K 2024-11-14T21:34:09,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/4133e34e9c994a50801fdd2097d6dfe3 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/4133e34e9c994a50801fdd2097d6dfe3 2024-11-14T21:34:09,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/4133e34e9c994a50801fdd2097d6dfe3, entries=150, sequenceid=61, filesize=11.7 K 2024-11-14T21:34:09,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/adc128f28a4d4cf79cb7671d14d58477 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/adc128f28a4d4cf79cb7671d14d58477 2024-11-14T21:34:09,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/adc128f28a4d4cf79cb7671d14d58477, entries=150, sequenceid=61, filesize=11.7 K 2024-11-14T21:34:09,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 81ffab6ef2cdc120372df3297622f922 in 309ms, sequenceid=61, compaction requested=true 2024-11-14T21:34:09,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:09,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:34:09,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:09,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:34:09,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:34:09,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:34:09,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:34:09,558 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:09,558 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:09,559 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:09,559 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/A is initiating minor compaction (all files) 2024-11-14T21:34:09,559 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/A in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:09,559 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/57dd4b418f42434d90bb4af6b642f938, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/cf48cf0160f447b3a26822591859c13c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/af210329842c4032983843b9353ecb9d] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=37.4 K 2024-11-14T21:34:09,560 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:09,560 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/B is initiating minor compaction (all files) 2024-11-14T21:34:09,560 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/B in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:09,560 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/17f68d20f8214dee9ae8a9cc6ed0a5f6, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/30ce30323cd44fa3a093982ee44bb095, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/4133e34e9c994a50801fdd2097d6dfe3] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=32.9 K 2024-11-14T21:34:09,560 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 57dd4b418f42434d90bb4af6b642f938, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731620048146 2024-11-14T21:34:09,561 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17f68d20f8214dee9ae8a9cc6ed0a5f6, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731620048204 2024-11-14T21:34:09,561 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting cf48cf0160f447b3a26822591859c13c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731620048424 2024-11-14T21:34:09,561 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30ce30323cd44fa3a093982ee44bb095, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731620048424 2024-11-14T21:34:09,561 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting af210329842c4032983843b9353ecb9d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731620049106 2024-11-14T21:34:09,562 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4133e34e9c994a50801fdd2097d6dfe3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731620049106 2024-11-14T21:34:09,587 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#A#compaction#383 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:09,588 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/7078a538412d4fe0a0f75d7b48570092 is 50, key is test_row_0/A:col10/1731620049106/Put/seqid=0 2024-11-14T21:34:09,608 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#B#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:09,609 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/ac4a2c44fea44765b2af673dac315f6d is 50, key is test_row_0/B:col10/1731620049106/Put/seqid=0 2024-11-14T21:34:09,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T21:34:09,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:09,615 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T21:34:09,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:09,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:09,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:09,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:09,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:09,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T21:34:09,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742281_1457 (size=12104) 2024-11-14T21:34:09,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/4312bc54578e412f9c679c6bac0587a3 is 50, key is test_row_0/A:col10/1731620049344/Put/seqid=0 2024-11-14T21:34:09,684 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/7078a538412d4fe0a0f75d7b48570092 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/7078a538412d4fe0a0f75d7b48570092 2024-11-14T21:34:09,689 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/A of 81ffab6ef2cdc120372df3297622f922 into 7078a538412d4fe0a0f75d7b48570092(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:09,689 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:09,690 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/A, priority=13, startTime=1731620049557; duration=0sec 2024-11-14T21:34:09,690 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:09,690 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:A 2024-11-14T21:34:09,690 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:09,693 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:09,693 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/C is initiating minor compaction (all files) 2024-11-14T21:34:09,693 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/C in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:09,693 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/fd7c5a8794da4daf9d4852e9c83ad136, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/db3368a49b914caa9b2096f5e33e2363, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/adc128f28a4d4cf79cb7671d14d58477] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=32.9 K 2024-11-14T21:34:09,703 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting fd7c5a8794da4daf9d4852e9c83ad136, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731620048204 2024-11-14T21:34:09,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742282_1458 (size=12104) 2024-11-14T21:34:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:09,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:09,709 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting db3368a49b914caa9b2096f5e33e2363, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731620048424 2024-11-14T21:34:09,711 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting adc128f28a4d4cf79cb7671d14d58477, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731620049106 2024-11-14T21:34:09,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742283_1459 (size=12001) 2024-11-14T21:34:09,738 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/4312bc54578e412f9c679c6bac0587a3 2024-11-14T21:34:09,769 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#C#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:09,770 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/d7fd73c9bded4ce4a5b548226e21a93a is 50, key is test_row_0/C:col10/1731620049106/Put/seqid=0 2024-11-14T21:34:09,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/cc9b54698e084b19bb47dd2b79948fcd is 50, key is test_row_0/B:col10/1731620049344/Put/seqid=0 2024-11-14T21:34:09,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620109755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620109761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620109763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620109764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620109788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742285_1461 (size=12001) 2024-11-14T21:34:09,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742284_1460 (size=12104) 2024-11-14T21:34:09,835 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/cc9b54698e084b19bb47dd2b79948fcd 2024-11-14T21:34:09,846 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/d7fd73c9bded4ce4a5b548226e21a93a as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/d7fd73c9bded4ce4a5b548226e21a93a 2024-11-14T21:34:09,852 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/C of 81ffab6ef2cdc120372df3297622f922 into d7fd73c9bded4ce4a5b548226e21a93a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:09,852 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:09,852 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/C, priority=13, startTime=1731620049558; duration=0sec 2024-11-14T21:34:09,852 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:09,853 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:C 2024-11-14T21:34:09,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/d2e98d9e1dd7489da2f4ecac8c14c9ec is 50, key is test_row_0/C:col10/1731620049344/Put/seqid=0 2024-11-14T21:34:09,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742286_1462 (size=12001) 2024-11-14T21:34:09,912 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/d2e98d9e1dd7489da2f4ecac8c14c9ec 2024-11-14T21:34:09,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/4312bc54578e412f9c679c6bac0587a3 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/4312bc54578e412f9c679c6bac0587a3 2024-11-14T21:34:09,928 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/4312bc54578e412f9c679c6bac0587a3, entries=150, sequenceid=80, filesize=11.7 K 2024-11-14T21:34:09,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/cc9b54698e084b19bb47dd2b79948fcd as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/cc9b54698e084b19bb47dd2b79948fcd 2024-11-14T21:34:09,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620109906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620109906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620109906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:09,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620109906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:09,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,936 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/cc9b54698e084b19bb47dd2b79948fcd, entries=150, sequenceid=80, filesize=11.7 K 2024-11-14T21:34:09,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/d2e98d9e1dd7489da2f4ecac8c14c9ec as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/d2e98d9e1dd7489da2f4ecac8c14c9ec 2024-11-14T21:34:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:09,942 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/d2e98d9e1dd7489da2f4ecac8c14c9ec, entries=150, sequenceid=80, filesize=11.7 K 2024-11-14T21:34:09,943 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 81ffab6ef2cdc120372df3297622f922 in 328ms, sequenceid=80, compaction requested=false 2024-11-14T21:34:09,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:09,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:09,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-14T21:34:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-14T21:34:09,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T21:34:09,950 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-14T21:34:09,950 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 646 msec 2024-11-14T21:34:09,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 651 msec 2024-11-14T21:34:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:09,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T21:34:09,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:09,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:09,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:09,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:09,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:09,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:09,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/14793bf660a147c69aae1dab4f01f49f is 50, key is test_row_0/A:col10/1731620049755/Put/seqid=0 2024-11-14T21:34:10,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742287_1463 (size=12001) 2024-11-14T21:34:10,121 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/ac4a2c44fea44765b2af673dac315f6d as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/ac4a2c44fea44765b2af673dac315f6d 2024-11-14T21:34:10,131 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/B of 81ffab6ef2cdc120372df3297622f922 into ac4a2c44fea44765b2af673dac315f6d(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:10,131 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:10,131 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/B, priority=13, startTime=1731620049558; duration=0sec 2024-11-14T21:34:10,131 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:10,131 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:B 2024-11-14T21:34:10,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620110182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:10,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:10,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620110191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:10,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620110196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:10,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620110230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:10,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620110231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:10,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,271 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,271 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:10,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620110341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:10,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620110341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:10,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620110341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:10,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:10,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620110373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:10,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:10,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620110380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:10,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/14793bf660a147c69aae1dab4f01f49f 2024-11-14T21:34:10,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/06f19271b69c4c3984d30a89ca2208ad is 50, key is test_row_0/B:col10/1731620049755/Put/seqid=0 2024-11-14T21:34:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T21:34:10,475 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:34:10,476 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:34:10,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-14T21:34:10,478 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:34:10,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T21:34:10,479 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:34:10,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:34:10,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742288_1464 (size=12001) 2024-11-14T21:34:10,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/06f19271b69c4c3984d30a89ca2208ad 2024-11-14T21:34:10,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/6ef91bb895af4c42aefe35c7cc7e5a71 is 50, key is test_row_0/C:col10/1731620049755/Put/seqid=0 2024-11-14T21:34:10,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620110562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T21:34:10,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620110562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:10,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620110570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:10,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:10,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742289_1465 (size=12001) 2024-11-14T21:34:10,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/6ef91bb895af4c42aefe35c7cc7e5a71 2024-11-14T21:34:10,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620110591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:10,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620110600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:10,620 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,620 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,620 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:10,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T21:34:10,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:10,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:10,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:10,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:10,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:10,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/14793bf660a147c69aae1dab4f01f49f as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/14793bf660a147c69aae1dab4f01f49f 2024-11-14T21:34:10,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/14793bf660a147c69aae1dab4f01f49f, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T21:34:10,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/06f19271b69c4c3984d30a89ca2208ad as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/06f19271b69c4c3984d30a89ca2208ad 2024-11-14T21:34:10,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/06f19271b69c4c3984d30a89ca2208ad, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T21:34:10,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/6ef91bb895af4c42aefe35c7cc7e5a71 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/6ef91bb895af4c42aefe35c7cc7e5a71 2024-11-14T21:34:10,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/6ef91bb895af4c42aefe35c7cc7e5a71, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T21:34:10,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 81ffab6ef2cdc120372df3297622f922 in 732ms, sequenceid=96, compaction requested=true 2024-11-14T21:34:10,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:10,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:34:10,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:10,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:34:10,697 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:10,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:10,697 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:10,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:34:10,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:10,708 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:10,708 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/A is initiating minor compaction (all files) 2024-11-14T21:34:10,709 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/A in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:10,709 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/7078a538412d4fe0a0f75d7b48570092, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/4312bc54578e412f9c679c6bac0587a3, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/14793bf660a147c69aae1dab4f01f49f] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=35.3 K 2024-11-14T21:34:10,709 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:10,709 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/B is initiating minor compaction (all files) 2024-11-14T21:34:10,709 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/B in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:10,709 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/ac4a2c44fea44765b2af673dac315f6d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/cc9b54698e084b19bb47dd2b79948fcd, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/06f19271b69c4c3984d30a89ca2208ad] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=35.3 K 2024-11-14T21:34:10,716 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7078a538412d4fe0a0f75d7b48570092, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731620049106 2024-11-14T21:34:10,716 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting ac4a2c44fea44765b2af673dac315f6d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731620049106 2024-11-14T21:34:10,717 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4312bc54578e412f9c679c6bac0587a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731620049326 2024-11-14T21:34:10,717 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting cc9b54698e084b19bb47dd2b79948fcd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731620049326 2024-11-14T21:34:10,723 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14793bf660a147c69aae1dab4f01f49f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731620049755 2024-11-14T21:34:10,724 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 06f19271b69c4c3984d30a89ca2208ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731620049755 2024-11-14T21:34:10,768 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#A#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:10,768 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/f627eaa85aa7459d961df4965118b6b6 is 50, key is test_row_0/A:col10/1731620049755/Put/seqid=0 2024-11-14T21:34:10,774 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#B#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:10,774 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/b23b83ce7c44459e9824323d712c0e3b is 50, key is test_row_0/B:col10/1731620049755/Put/seqid=0 2024-11-14T21:34:10,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T21:34:10,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T21:34:10,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:10,801 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T21:34:10,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:10,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:10,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:10,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:10,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:10,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:10,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742290_1466 (size=12207) 2024-11-14T21:34:10,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/4491d6dd39dc4976a3b75600bf70e951 is 50, key is test_row_0/A:col10/1731620050181/Put/seqid=0 2024-11-14T21:34:10,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742291_1467 (size=12207) 2024-11-14T21:34:10,829 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/f627eaa85aa7459d961df4965118b6b6 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/f627eaa85aa7459d961df4965118b6b6 2024-11-14T21:34:10,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742292_1468 (size=12001) 2024-11-14T21:34:10,839 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/A of 81ffab6ef2cdc120372df3297622f922 into f627eaa85aa7459d961df4965118b6b6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:10,839 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:10,839 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/A, priority=13, startTime=1731620050697; duration=0sec 2024-11-14T21:34:10,839 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:10,839 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:A 2024-11-14T21:34:10,839 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:10,840 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/4491d6dd39dc4976a3b75600bf70e951 2024-11-14T21:34:10,841 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:10,841 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/C is initiating minor compaction (all files) 2024-11-14T21:34:10,841 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/C in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:10,841 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/d7fd73c9bded4ce4a5b548226e21a93a, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/d2e98d9e1dd7489da2f4ecac8c14c9ec, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/6ef91bb895af4c42aefe35c7cc7e5a71] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=35.3 K 2024-11-14T21:34:10,842 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7fd73c9bded4ce4a5b548226e21a93a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731620049106 2024-11-14T21:34:10,842 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2e98d9e1dd7489da2f4ecac8c14c9ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731620049326 2024-11-14T21:34:10,843 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ef91bb895af4c42aefe35c7cc7e5a71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731620049755 2024-11-14T21:34:10,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/8c37d070cd80475b892a299d85b00f86 is 50, key is test_row_0/B:col10/1731620050181/Put/seqid=0 2024-11-14T21:34:10,864 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#C#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:10,865 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/b23ea78fea0c4873822cc12777c2f1b4 is 50, key is test_row_0/C:col10/1731620049755/Put/seqid=0 2024-11-14T21:34:10,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:10,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:10,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742293_1469 (size=12001) 2024-11-14T21:34:10,908 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/8c37d070cd80475b892a299d85b00f86 2024-11-14T21:34:10,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742294_1470 (size=12207) 2024-11-14T21:34:10,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/69b3ea5b687f4ef685815e042f171aa2 is 50, key is test_row_0/C:col10/1731620050181/Put/seqid=0 2024-11-14T21:34:10,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:10,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620110935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:10,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:10,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:10,962 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/b23ea78fea0c4873822cc12777c2f1b4 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/b23ea78fea0c4873822cc12777c2f1b4 2024-11-14T21:34:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620110948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:10,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:10,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620110952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:10,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:10,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620110955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:10,975 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,975 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,975 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:10,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:10,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620110961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:10,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742295_1471 (size=12001) 2024-11-14T21:34:10,996 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/69b3ea5b687f4ef685815e042f171aa2 2024-11-14T21:34:10,997 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/C of 81ffab6ef2cdc120372df3297622f922 into b23ea78fea0c4873822cc12777c2f1b4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:10,997 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:10,997 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/C, priority=13, startTime=1731620050697; duration=0sec 2024-11-14T21:34:10,997 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:10,997 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:C 2024-11-14T21:34:11,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/4491d6dd39dc4976a3b75600bf70e951 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/4491d6dd39dc4976a3b75600bf70e951 2024-11-14T21:34:11,060 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/4491d6dd39dc4976a3b75600bf70e951, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T21:34:11,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/8c37d070cd80475b892a299d85b00f86 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/8c37d070cd80475b892a299d85b00f86 2024-11-14T21:34:11,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620111073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:11,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620111073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:11,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:11,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:11,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T21:34:11,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620111085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:11,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620111085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:11,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:11,110 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/8c37d070cd80475b892a299d85b00f86, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T21:34:11,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620111090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:11,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/69b3ea5b687f4ef685815e042f171aa2 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/69b3ea5b687f4ef685815e042f171aa2 2024-11-14T21:34:11,144 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/69b3ea5b687f4ef685815e042f171aa2, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T21:34:11,145 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 81ffab6ef2cdc120372df3297622f922 in 344ms, sequenceid=116, compaction requested=false 2024-11-14T21:34:11,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:11,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:11,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-14T21:34:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-14T21:34:11,149 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-14T21:34:11,149 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 668 msec 2024-11-14T21:34:11,151 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 673 msec 2024-11-14T21:34:11,278 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/b23b83ce7c44459e9824323d712c0e3b as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/b23b83ce7c44459e9824323d712c0e3b 2024-11-14T21:34:11,290 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/B of 81ffab6ef2cdc120372df3297622f922 into b23b83ce7c44459e9824323d712c0e3b(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:11,290 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:11,290 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/B, priority=13, startTime=1731620050697; duration=0sec 2024-11-14T21:34:11,290 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:11,290 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:B 2024-11-14T21:34:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:11,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T21:34:11,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:11,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:11,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:11,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:11,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:11,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:11,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/a98862afab4840858cb4fdeb26a59642 is 50, key is test_row_0/A:col10/1731620050946/Put/seqid=0 2024-11-14T21:34:11,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742296_1472 (size=14441) 2024-11-14T21:34:11,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:11,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620111342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:11,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:11,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:11,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620111344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:11,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:11,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:11,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620111358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:11,378 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:11,378 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,378 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620111360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:11,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:11,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620111361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:11,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:11,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620111470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:11,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620111471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:11,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:11,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:11,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620111481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:11,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620111481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:11,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:11,499 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,499 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,499 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:11,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620111493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:11,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T21:34:11,613 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:34:11,616 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:34:11,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-14T21:34:11,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T21:34:11,622 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:34:11,623 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:34:11,623 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:34:11,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620111691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:11,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:11,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620111691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:11,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620111703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620111706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:11,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:11,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620111710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:11,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:11,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T21:34:11,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/a98862afab4840858cb4fdeb26a59642 2024-11-14T21:34:11,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/c5a92250670e4f6394bcde64de4234d1 is 50, key is test_row_0/B:col10/1731620050946/Put/seqid=0 2024-11-14T21:34:11,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T21:34:11,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:11,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:11,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:11,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:11,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:11,780 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T21:34:11,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742297_1473 (size=12101) 2024-11-14T21:34:11,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/c5a92250670e4f6394bcde64de4234d1 2024-11-14T21:34:11,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/0938635633a64653b27523d97b7140d7 is 50, key is test_row_0/C:col10/1731620050946/Put/seqid=0 2024-11-14T21:34:11,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T21:34:11,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:11,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:11,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:11,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:11,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:11,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742298_1474 (size=12101) 2024-11-14T21:34:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T21:34:12,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620112011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:12,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:12,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620112014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:12,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:12,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620112011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:12,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620112020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:12,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:12,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620112032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:12,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:12,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T21:34:12,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:12,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:12,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:12,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:12,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:12,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T21:34:12,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:12,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:12,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:12,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:12,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T21:34:12,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/0938635633a64653b27523d97b7140d7 2024-11-14T21:34:12,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/a98862afab4840858cb4fdeb26a59642 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/a98862afab4840858cb4fdeb26a59642 2024-11-14T21:34:12,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/a98862afab4840858cb4fdeb26a59642, entries=200, sequenceid=133, filesize=14.1 K 2024-11-14T21:34:12,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/c5a92250670e4f6394bcde64de4234d1 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/c5a92250670e4f6394bcde64de4234d1 2024-11-14T21:34:12,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/c5a92250670e4f6394bcde64de4234d1, entries=150, sequenceid=133, filesize=11.8 K 2024-11-14T21:34:12,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/0938635633a64653b27523d97b7140d7 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/0938635633a64653b27523d97b7140d7 2024-11-14T21:34:12,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/0938635633a64653b27523d97b7140d7, entries=150, sequenceid=133, filesize=11.8 K 2024-11-14T21:34:12,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 81ffab6ef2cdc120372df3297622f922 in 1100ms, sequenceid=133, compaction requested=true 2024-11-14T21:34:12,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:12,398 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:12,399 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:12,399 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/A is initiating minor compaction (all files) 2024-11-14T21:34:12,399 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/A in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:12,399 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/f627eaa85aa7459d961df4965118b6b6, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/4491d6dd39dc4976a3b75600bf70e951, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/a98862afab4840858cb4fdeb26a59642] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=37.7 K 2024-11-14T21:34:12,399 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting f627eaa85aa7459d961df4965118b6b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731620049755 2024-11-14T21:34:12,400 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4491d6dd39dc4976a3b75600bf70e951, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731620050181 2024-11-14T21:34:12,400 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting a98862afab4840858cb4fdeb26a59642, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731620050946 2024-11-14T21:34:12,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:34:12,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:12,410 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:12,411 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:12,411 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/B is initiating minor compaction (all files) 2024-11-14T21:34:12,411 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/B in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:12,411 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/b23b83ce7c44459e9824323d712c0e3b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/8c37d070cd80475b892a299d85b00f86, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/c5a92250670e4f6394bcde64de4234d1] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=35.5 K 2024-11-14T21:34:12,411 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting b23b83ce7c44459e9824323d712c0e3b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731620049755 2024-11-14T21:34:12,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:34:12,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:12,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:34:12,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:12,412 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c37d070cd80475b892a299d85b00f86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731620050181 2024-11-14T21:34:12,413 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#A#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:12,414 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/e600a24bb50e4b3e95ce896c75c0ee39 is 50, key is test_row_0/A:col10/1731620050946/Put/seqid=0 2024-11-14T21:34:12,414 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting c5a92250670e4f6394bcde64de4234d1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731620050946 2024-11-14T21:34:12,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T21:34:12,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:12,419 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T21:34:12,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:12,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:12,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:12,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:12,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:12,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:12,425 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#B#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:12,426 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/df8c19d2d8644e379e48e317c746628e is 50, key is test_row_0/B:col10/1731620050946/Put/seqid=0 2024-11-14T21:34:12,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742299_1475 (size=12409) 2024-11-14T21:34:12,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/65120ba626ae43a59c908676787ca62d is 50, key is test_row_0/A:col10/1731620051321/Put/seqid=0 2024-11-14T21:34:12,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742300_1476 (size=12409) 2024-11-14T21:34:12,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742301_1477 (size=12151) 2024-11-14T21:34:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:12,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:12,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620112575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:12,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620112576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:12,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:12,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620112578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:12,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:12,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:12,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620112598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:12,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:12,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620112602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:12,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:12,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:12,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620112711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:12,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:12,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620112711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:12,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:12,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:12,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:12,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620112714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:12,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:12,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:12,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620112730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:12,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:12,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:12,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620112730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:12,745 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,745 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,745 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T21:34:12,836 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/e600a24bb50e4b3e95ce896c75c0ee39 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/e600a24bb50e4b3e95ce896c75c0ee39 2024-11-14T21:34:12,841 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/A of 81ffab6ef2cdc120372df3297622f922 into e600a24bb50e4b3e95ce896c75c0ee39(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:12,841 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:12,841 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/A, priority=13, startTime=1731620052398; duration=0sec 2024-11-14T21:34:12,841 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:12,841 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:A 2024-11-14T21:34:12,841 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:12,842 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:12,842 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/C is initiating minor compaction (all files) 2024-11-14T21:34:12,842 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/C in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:12,842 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/b23ea78fea0c4873822cc12777c2f1b4, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/69b3ea5b687f4ef685815e042f171aa2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/0938635633a64653b27523d97b7140d7] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=35.5 K 2024-11-14T21:34:12,843 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting b23ea78fea0c4873822cc12777c2f1b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731620049755 2024-11-14T21:34:12,843 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69b3ea5b687f4ef685815e042f171aa2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731620050181 2024-11-14T21:34:12,843 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0938635633a64653b27523d97b7140d7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731620050946 2024-11-14T21:34:12,855 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#C#compaction#404 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:12,856 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/a7fb991a6ecd43d79ec85fb17056b30e is 50, key is test_row_0/C:col10/1731620050946/Put/seqid=0 2024-11-14T21:34:12,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742302_1478 (size=12409) 2024-11-14T21:34:12,866 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/a7fb991a6ecd43d79ec85fb17056b30e as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/a7fb991a6ecd43d79ec85fb17056b30e 2024-11-14T21:34:12,876 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/C of 81ffab6ef2cdc120372df3297622f922 into a7fb991a6ecd43d79ec85fb17056b30e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:12,876 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:12,876 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/C, priority=13, startTime=1731620052412; duration=0sec 2024-11-14T21:34:12,876 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:12,876 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:C 2024-11-14T21:34:12,910 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/df8c19d2d8644e379e48e317c746628e as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/df8c19d2d8644e379e48e317c746628e 2024-11-14T21:34:12,914 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/B of 81ffab6ef2cdc120372df3297622f922 into df8c19d2d8644e379e48e317c746628e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:12,914 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:12,914 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/B, priority=13, startTime=1731620052409; duration=0sec 2024-11-14T21:34:12,914 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:12,914 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:B 2024-11-14T21:34:12,925 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/65120ba626ae43a59c908676787ca62d 2024-11-14T21:34:12,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:12,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620112931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:12,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:12,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620112932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:12,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/a50b594c28ed4635b8d33fd3aac41ccc is 50, key is test_row_0/B:col10/1731620051321/Put/seqid=0 2024-11-14T21:34:12,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:12,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620112941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:12,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:12,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620112950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:12,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620112950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:12,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:12,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:12,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:13,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742303_1479 (size=12151) 2024-11-14T21:34:13,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:13,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620113251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:13,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:13,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620113252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:13,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:13,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:13,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:13,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620113252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:13,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:13,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:13,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:13,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:13,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:13,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:13,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:13,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:13,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620113281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:13,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:13,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:13,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:13,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:13,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620113281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:13,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:13,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:13,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:13,406 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/a50b594c28ed4635b8d33fd3aac41ccc 2024-11-14T21:34:13,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/cc091a71363d44779c64ba5c28208137 is 50, key is test_row_0/C:col10/1731620051321/Put/seqid=0 2024-11-14T21:34:13,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742304_1480 (size=12151) 2024-11-14T21:34:13,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T21:34:13,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:13,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620113775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:13,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:13,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620113779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:13,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:13,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:13,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620113783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:13,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:13,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:13,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:13,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:13,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620113811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:13,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:13,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620113812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:13,873 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/cc091a71363d44779c64ba5c28208137 2024-11-14T21:34:13,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/65120ba626ae43a59c908676787ca62d as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/65120ba626ae43a59c908676787ca62d 2024-11-14T21:34:13,911 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/65120ba626ae43a59c908676787ca62d, entries=150, sequenceid=151, filesize=11.9 K 2024-11-14T21:34:13,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/a50b594c28ed4635b8d33fd3aac41ccc as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/a50b594c28ed4635b8d33fd3aac41ccc 2024-11-14T21:34:13,919 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/a50b594c28ed4635b8d33fd3aac41ccc, entries=150, sequenceid=151, filesize=11.9 K 2024-11-14T21:34:13,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/cc091a71363d44779c64ba5c28208137 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/cc091a71363d44779c64ba5c28208137 2024-11-14T21:34:13,925 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/cc091a71363d44779c64ba5c28208137, entries=150, sequenceid=151, filesize=11.9 K 2024-11-14T21:34:13,926 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 81ffab6ef2cdc120372df3297622f922 in 1506ms, sequenceid=151, compaction requested=false 2024-11-14T21:34:13,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:13,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:13,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-14T21:34:13,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-14T21:34:13,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-14T21:34:13,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3050 sec 2024-11-14T21:34:13,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.3140 sec 2024-11-14T21:34:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:14,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T21:34:14,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:14,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:14,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:14,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:14,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:14,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:14,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/65d89ef0b9fb46b5af4233bdb90babdb is 50, key is test_row_0/A:col10/1731620054805/Put/seqid=0 2024-11-14T21:34:14,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742305_1481 (size=12151) 2024-11-14T21:34:14,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/65d89ef0b9fb46b5af4233bdb90babdb 2024-11-14T21:34:14,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/ed100804e0954c4fa6bf2d5b994dec73 is 50, key is test_row_0/B:col10/1731620054805/Put/seqid=0 2024-11-14T21:34:14,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620114866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:14,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:14,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:14,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:14,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620114875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:14,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620114875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:14,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620114879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:14,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:14,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:14,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:14,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620114884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:14,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:14,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:14,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:14,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742306_1482 (size=12151) 2024-11-14T21:34:14,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/ed100804e0954c4fa6bf2d5b994dec73 2024-11-14T21:34:14,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/59730e81ae344195855836053b7c13d7 is 50, key is test_row_0/C:col10/1731620054805/Put/seqid=0 2024-11-14T21:34:14,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742307_1483 (size=12151) 2024-11-14T21:34:15,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620114991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620115001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620115001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620115010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620115022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620115211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620115211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620115211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620115231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620115234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/59730e81ae344195855836053b7c13d7 2024-11-14T21:34:15,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/65d89ef0b9fb46b5af4233bdb90babdb as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/65d89ef0b9fb46b5af4233bdb90babdb 2024-11-14T21:34:15,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/65d89ef0b9fb46b5af4233bdb90babdb, entries=150, sequenceid=170, filesize=11.9 K 2024-11-14T21:34:15,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/ed100804e0954c4fa6bf2d5b994dec73 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/ed100804e0954c4fa6bf2d5b994dec73 2024-11-14T21:34:15,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/ed100804e0954c4fa6bf2d5b994dec73, entries=150, sequenceid=170, filesize=11.9 K 2024-11-14T21:34:15,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/59730e81ae344195855836053b7c13d7 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/59730e81ae344195855836053b7c13d7 2024-11-14T21:34:15,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/59730e81ae344195855836053b7c13d7, entries=150, sequenceid=170, filesize=11.9 K 2024-11-14T21:34:15,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 81ffab6ef2cdc120372df3297622f922 in 646ms, sequenceid=170, compaction requested=true 2024-11-14T21:34:15,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:15,459 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:15,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:34:15,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:15,459 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:15,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:34:15,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:15,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:34:15,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:15,465 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:15,465 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/A is initiating minor compaction (all files) 2024-11-14T21:34:15,465 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/A in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:15,466 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/e600a24bb50e4b3e95ce896c75c0ee39, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/65120ba626ae43a59c908676787ca62d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/65d89ef0b9fb46b5af4233bdb90babdb] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=35.9 K 2024-11-14T21:34:15,466 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:15,466 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/B is initiating minor compaction (all files) 2024-11-14T21:34:15,466 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/B in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:15,466 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/df8c19d2d8644e379e48e317c746628e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/a50b594c28ed4635b8d33fd3aac41ccc, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/ed100804e0954c4fa6bf2d5b994dec73] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=35.9 K 2024-11-14T21:34:15,466 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting e600a24bb50e4b3e95ce896c75c0ee39, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731620050946 2024-11-14T21:34:15,467 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting df8c19d2d8644e379e48e317c746628e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731620050946 2024-11-14T21:34:15,467 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65120ba626ae43a59c908676787ca62d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731620051321 2024-11-14T21:34:15,467 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting a50b594c28ed4635b8d33fd3aac41ccc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731620051321 2024-11-14T21:34:15,467 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65d89ef0b9fb46b5af4233bdb90babdb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731620052574 2024-11-14T21:34:15,467 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting ed100804e0954c4fa6bf2d5b994dec73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731620052574 2024-11-14T21:34:15,475 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#A#compaction#410 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:15,476 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/9b9dc472f9a7438bb6ee2ca1ee387827 is 50, key is test_row_0/A:col10/1731620054805/Put/seqid=0 2024-11-14T21:34:15,477 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#B#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:15,477 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/54df9099bfd0459bb35800ad3c436f92 is 50, key is test_row_0/B:col10/1731620054805/Put/seqid=0 2024-11-14T21:34:15,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742308_1484 (size=12561) 2024-11-14T21:34:15,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742309_1485 (size=12561) 2024-11-14T21:34:15,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T21:34:15,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:15,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:15,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:15,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:15,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:15,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:15,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/661ad75f841a408493c99b62dd7c8aac is 50, key is test_row_0/A:col10/1731620055538/Put/seqid=0 2024-11-14T21:34:15,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742310_1486 (size=12151) 2024-11-14T21:34:15,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620115590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620115590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620115592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620115593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620115592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620115722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620115722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620115722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620115723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620115723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T21:34:15,814 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:34:15,819 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:34:15,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-14T21:34:15,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T21:34:15,830 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:34:15,832 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:34:15,832 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:34:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T21:34:15,935 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/9b9dc472f9a7438bb6ee2ca1ee387827 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/9b9dc472f9a7438bb6ee2ca1ee387827 2024-11-14T21:34:15,952 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/54df9099bfd0459bb35800ad3c436f92 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/54df9099bfd0459bb35800ad3c436f92 2024-11-14T21:34:15,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620115943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620115943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620115943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620115946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620115950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:15,962 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/A of 81ffab6ef2cdc120372df3297622f922 into 9b9dc472f9a7438bb6ee2ca1ee387827(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:15,962 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:15,962 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/A, priority=13, startTime=1731620055458; duration=0sec 2024-11-14T21:34:15,962 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:15,962 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:A 2024-11-14T21:34:15,962 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:15,966 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/B of 81ffab6ef2cdc120372df3297622f922 into 54df9099bfd0459bb35800ad3c436f92(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:15,966 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:15,966 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/B, priority=13, startTime=1731620055459; duration=0sec 2024-11-14T21:34:15,966 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:15,966 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:B 2024-11-14T21:34:15,970 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:15,971 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/C is initiating minor compaction (all files) 2024-11-14T21:34:15,971 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/C in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:15,971 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/a7fb991a6ecd43d79ec85fb17056b30e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/cc091a71363d44779c64ba5c28208137, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/59730e81ae344195855836053b7c13d7] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=35.9 K 2024-11-14T21:34:15,974 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7fb991a6ecd43d79ec85fb17056b30e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731620050946 2024-11-14T21:34:15,980 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc091a71363d44779c64ba5c28208137, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731620051321 2024-11-14T21:34:15,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/661ad75f841a408493c99b62dd7c8aac 2024-11-14T21:34:15,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T21:34:15,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:15,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:15,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:15,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:15,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:15,987 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59730e81ae344195855836053b7c13d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731620052574 2024-11-14T21:34:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:15,999 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#C#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:16,000 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/ff05576ebd8744238de7d321ac14522c is 50, key is test_row_0/C:col10/1731620054805/Put/seqid=0 2024-11-14T21:34:16,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/43cefb633d614dfd8b53d8e93a4d71ab is 50, key is test_row_0/B:col10/1731620055538/Put/seqid=0 2024-11-14T21:34:16,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742311_1487 (size=12561) 2024-11-14T21:34:16,053 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/ff05576ebd8744238de7d321ac14522c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/ff05576ebd8744238de7d321ac14522c 2024-11-14T21:34:16,059 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/C of 81ffab6ef2cdc120372df3297622f922 into ff05576ebd8744238de7d321ac14522c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:16,059 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:16,059 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/C, priority=13, startTime=1731620055460; duration=0sec 2024-11-14T21:34:16,059 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:16,059 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:C 2024-11-14T21:34:16,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742312_1488 (size=12151) 2024-11-14T21:34:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T21:34:16,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T21:34:16,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:16,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:16,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:16,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:16,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:16,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:16,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620116260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:16,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620116260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:16,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620116271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:16,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:16,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620116271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:16,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620116271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:16,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:16,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T21:34:16,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:16,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:16,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:16,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:16,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T21:34:16,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T21:34:16,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:16,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:16,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:16,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:16,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:16,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/43cefb633d614dfd8b53d8e93a4d71ab 2024-11-14T21:34:16,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/543eb1d6495849c39acaa06d9cdb849f is 50, key is test_row_0/C:col10/1731620055538/Put/seqid=0 2024-11-14T21:34:16,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742313_1489 (size=12151) 2024-11-14T21:34:16,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/543eb1d6495849c39acaa06d9cdb849f 2024-11-14T21:34:16,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/661ad75f841a408493c99b62dd7c8aac as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/661ad75f841a408493c99b62dd7c8aac 2024-11-14T21:34:16,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/661ad75f841a408493c99b62dd7c8aac, entries=150, sequenceid=188, filesize=11.9 K 2024-11-14T21:34:16,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/43cefb633d614dfd8b53d8e93a4d71ab as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/43cefb633d614dfd8b53d8e93a4d71ab 2024-11-14T21:34:16,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T21:34:16,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:16,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:16,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:16,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:16,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:16,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/43cefb633d614dfd8b53d8e93a4d71ab, entries=150, sequenceid=188, filesize=11.9 K 2024-11-14T21:34:16,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/543eb1d6495849c39acaa06d9cdb849f as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/543eb1d6495849c39acaa06d9cdb849f 2024-11-14T21:34:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:16,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/543eb1d6495849c39acaa06d9cdb849f, entries=150, sequenceid=188, filesize=11.9 K 2024-11-14T21:34:16,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 81ffab6ef2cdc120372df3297622f922 in 1095ms, sequenceid=188, compaction requested=false 2024-11-14T21:34:16,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:16,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T21:34:16,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:16,778 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T21:34:16,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:16,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:16,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:16,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:16,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:16,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:16,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/268d1809fc3c4fa68a7fae07abf943f3 is 50, key is test_row_0/A:col10/1731620055591/Put/seqid=0 2024-11-14T21:34:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:16,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:16,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742314_1490 (size=12151) 2024-11-14T21:34:16,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620116811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:16,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620116832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:16,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620116833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:16,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:16,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620116834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:16,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620116837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:16,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:16,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:16,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620116941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:16,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:16,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620116950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:16,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620116950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T21:34:16,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620116961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:16,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:16,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620116961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:16,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:16,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:16,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:17,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:17,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620117152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:17,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620117160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:17,166 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:17,166 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:17,166 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:17,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620117162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:17,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:17,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:17,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:17,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620117181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:17,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620117189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:17,201 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/268d1809fc3c4fa68a7fae07abf943f3 2024-11-14T21:34:17,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/5f69c66ed6cc45abafa85332985e5fce is 50, key is test_row_0/B:col10/1731620055591/Put/seqid=0 2024-11-14T21:34:17,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742315_1491 (size=12151) 2024-11-14T21:34:17,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:17,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620117471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:17,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:17,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620117471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:17,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:17,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620117495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:17,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:17,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:17,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:17,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:17,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620117500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:17,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:17,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620117501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:17,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:17,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:17,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:17,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:17,723 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/5f69c66ed6cc45abafa85332985e5fce 2024-11-14T21:34:17,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/83b35f08b57e4377ba18b6f742f91495 is 50, key is test_row_0/C:col10/1731620055591/Put/seqid=0 2024-11-14T21:34:17,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742316_1492 (size=12151) 2024-11-14T21:34:17,851 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/83b35f08b57e4377ba18b6f742f91495 2024-11-14T21:34:17,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/268d1809fc3c4fa68a7fae07abf943f3 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/268d1809fc3c4fa68a7fae07abf943f3 2024-11-14T21:34:17,895 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/268d1809fc3c4fa68a7fae07abf943f3, entries=150, sequenceid=205, filesize=11.9 K 2024-11-14T21:34:17,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/5f69c66ed6cc45abafa85332985e5fce as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/5f69c66ed6cc45abafa85332985e5fce 2024-11-14T21:34:17,912 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/5f69c66ed6cc45abafa85332985e5fce, entries=150, sequenceid=205, filesize=11.9 K 2024-11-14T21:34:17,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/83b35f08b57e4377ba18b6f742f91495 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/83b35f08b57e4377ba18b6f742f91495 2024-11-14T21:34:17,941 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/83b35f08b57e4377ba18b6f742f91495, entries=150, sequenceid=205, filesize=11.9 K 2024-11-14T21:34:17,942 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 81ffab6ef2cdc120372df3297622f922 in 1165ms, sequenceid=205, compaction requested=true 2024-11-14T21:34:17,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:17,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:17,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-14T21:34:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-14T21:34:17,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-14T21:34:17,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1190 sec 2024-11-14T21:34:17,970 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.1410 sec 2024-11-14T21:34:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T21:34:17,982 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:34:17,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:34:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-14T21:34:17,991 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:34:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T21:34:17,996 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:34:17,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:34:18,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T21:34:18,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:18,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:18,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:18,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:18,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:18,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:18,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/066961be56ee46389d5fc5f390918539 is 50, key is test_row_0/A:col10/1731620058001/Put/seqid=0 2024-11-14T21:34:18,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742317_1493 (size=14541) 2024-11-14T21:34:18,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620118037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:18,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620118049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:18,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620118052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:18,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:18,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620118057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:18,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:18,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:18,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620118059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T21:34:18,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T21:34:18,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:18,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:18,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:18,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:18,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:18,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620118175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:18,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:18,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620118183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:18,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:18,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620118183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:18,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:18,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620118183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:18,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:18,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620118190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:18,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T21:34:18,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T21:34:18,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:18,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:18,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:18,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:18,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:18,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620118390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:18,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620118390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620118390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:18,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620118390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:18,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:18,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620118421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:18,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/066961be56ee46389d5fc5f390918539 2024-11-14T21:34:18,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/a0c3691791594713bdbf9e055832a150 is 50, key is test_row_0/B:col10/1731620058001/Put/seqid=0 2024-11-14T21:34:18,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T21:34:18,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:18,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:18,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:18,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:18,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:18,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742318_1494 (size=12151) 2024-11-14T21:34:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:18,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T21:34:18,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T21:34:18,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:18,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:18,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:18,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:18,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:18,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620118711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:18,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620118711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:18,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620118711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:18,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620118711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:18,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:18,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:18,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:18,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620118731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:18,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:18,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:18,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T21:34:18,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:18,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:18,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:18,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:18,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:18,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/a0c3691791594713bdbf9e055832a150 2024-11-14T21:34:18,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/f6301a7c18d243308790cf649be28464 is 50, key is test_row_0/C:col10/1731620058001/Put/seqid=0 2024-11-14T21:34:18,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742319_1495 (size=12151) 2024-11-14T21:34:18,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/f6301a7c18d243308790cf649be28464 2024-11-14T21:34:18,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/066961be56ee46389d5fc5f390918539 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/066961be56ee46389d5fc5f390918539 2024-11-14T21:34:18,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T21:34:18,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:18,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:18,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:18,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:18,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:18,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/066961be56ee46389d5fc5f390918539, entries=200, sequenceid=224, filesize=14.2 K 2024-11-14T21:34:18,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/a0c3691791594713bdbf9e055832a150 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/a0c3691791594713bdbf9e055832a150 2024-11-14T21:34:18,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/a0c3691791594713bdbf9e055832a150, entries=150, sequenceid=224, filesize=11.9 K 2024-11-14T21:34:18,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/f6301a7c18d243308790cf649be28464 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/f6301a7c18d243308790cf649be28464 2024-11-14T21:34:18,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/f6301a7c18d243308790cf649be28464, entries=150, sequenceid=224, filesize=11.9 K 2024-11-14T21:34:19,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 81ffab6ef2cdc120372df3297622f922 in 1002ms, sequenceid=224, compaction requested=true 2024-11-14T21:34:19,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:19,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:34:19,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:19,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:34:19,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:19,005 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:34:19,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:34:19,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:34:19,005 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:34:19,006 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51404 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:34:19,006 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/A is initiating minor compaction (all files) 2024-11-14T21:34:19,006 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/A in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:19,006 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/9b9dc472f9a7438bb6ee2ca1ee387827, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/661ad75f841a408493c99b62dd7c8aac, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/268d1809fc3c4fa68a7fae07abf943f3, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/066961be56ee46389d5fc5f390918539] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=50.2 K 2024-11-14T21:34:19,006 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:34:19,006 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/B is initiating minor compaction (all files) 2024-11-14T21:34:19,006 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/B in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:19,007 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/54df9099bfd0459bb35800ad3c436f92, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/43cefb633d614dfd8b53d8e93a4d71ab, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/5f69c66ed6cc45abafa85332985e5fce, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/a0c3691791594713bdbf9e055832a150] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=47.9 K 2024-11-14T21:34:19,007 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b9dc472f9a7438bb6ee2ca1ee387827, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731620052574 2024-11-14T21:34:19,007 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 54df9099bfd0459bb35800ad3c436f92, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731620052574 2024-11-14T21:34:19,007 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 661ad75f841a408493c99b62dd7c8aac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731620054847 2024-11-14T21:34:19,008 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 43cefb633d614dfd8b53d8e93a4d71ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731620054847 2024-11-14T21:34:19,008 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 268d1809fc3c4fa68a7fae07abf943f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731620055590 2024-11-14T21:34:19,008 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f69c66ed6cc45abafa85332985e5fce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731620055590 2024-11-14T21:34:19,008 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 066961be56ee46389d5fc5f390918539, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731620056804 2024-11-14T21:34:19,008 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting a0c3691791594713bdbf9e055832a150, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731620056804 2024-11-14T21:34:19,025 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#B#compaction#422 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:19,026 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/097fbac7b96943738c819d5c90ce7d12 is 50, key is test_row_0/B:col10/1731620058001/Put/seqid=0 2024-11-14T21:34:19,030 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#A#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:19,031 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/1982b06e04064c6fab6405ee0babc2a2 is 50, key is test_row_0/A:col10/1731620058001/Put/seqid=0 2024-11-14T21:34:19,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742320_1496 (size=12697) 2024-11-14T21:34:19,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742321_1497 (size=12697) 2024-11-14T21:34:19,090 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/1982b06e04064c6fab6405ee0babc2a2 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/1982b06e04064c6fab6405ee0babc2a2 2024-11-14T21:34:19,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T21:34:19,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:19,117 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T21:34:19,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:19,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:19,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:19,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:19,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:19,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:19,123 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/A of 81ffab6ef2cdc120372df3297622f922 into 1982b06e04064c6fab6405ee0babc2a2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:19,123 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:19,123 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/A, priority=12, startTime=1731620059004; duration=0sec 2024-11-14T21:34:19,123 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:19,123 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:A 2024-11-14T21:34:19,123 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:34:19,131 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:34:19,131 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/C is initiating minor compaction (all files) 2024-11-14T21:34:19,131 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/C in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:19,132 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/ff05576ebd8744238de7d321ac14522c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/543eb1d6495849c39acaa06d9cdb849f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/83b35f08b57e4377ba18b6f742f91495, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/f6301a7c18d243308790cf649be28464] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=47.9 K 2024-11-14T21:34:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T21:34:19,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/840c1c8a5dcb4c9184d70a2b0d5d0ed0 is 50, key is test_row_0/A:col10/1731620058033/Put/seqid=0 2024-11-14T21:34:19,137 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff05576ebd8744238de7d321ac14522c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731620052574 2024-11-14T21:34:19,138 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 543eb1d6495849c39acaa06d9cdb849f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731620054847 2024-11-14T21:34:19,143 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83b35f08b57e4377ba18b6f742f91495, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731620055590 2024-11-14T21:34:19,143 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6301a7c18d243308790cf649be28464, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731620056804 2024-11-14T21:34:19,184 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#C#compaction#425 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:19,184 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/43941c6e16b14074889b575104ba8bb8 is 50, key is test_row_0/C:col10/1731620058001/Put/seqid=0 2024-11-14T21:34:19,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742322_1498 (size=12151) 2024-11-14T21:34:19,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742323_1499 (size=12697) 2024-11-14T21:34:19,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:19,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620119305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:19,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620119309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:19,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620119311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:19,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620119313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:19,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:19,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620119315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:19,455 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/097fbac7b96943738c819d5c90ce7d12 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/097fbac7b96943738c819d5c90ce7d12 2024-11-14T21:34:19,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620119441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:19,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620119441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:19,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:19,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:19,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620119442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:19,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:19,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620119442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:19,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:19,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620119442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:19,473 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/B of 81ffab6ef2cdc120372df3297622f922 into 097fbac7b96943738c819d5c90ce7d12(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:19,473 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:19,473 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/B, priority=12, startTime=1731620059004; duration=0sec 2024-11-14T21:34:19,473 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:19,473 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:B 2024-11-14T21:34:19,604 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/840c1c8a5dcb4c9184d70a2b0d5d0ed0 2024-11-14T21:34:19,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/e583a1d49ca7408c93dd22048ddbcd99 is 50, key is test_row_0/B:col10/1731620058033/Put/seqid=0 2024-11-14T21:34:19,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742324_1500 (size=12151) 2024-11-14T21:34:19,620 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/e583a1d49ca7408c93dd22048ddbcd99 2024-11-14T21:34:19,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/05efc3a2769a4c2e8eae37cd5139b932 is 50, key is test_row_0/C:col10/1731620058033/Put/seqid=0 2024-11-14T21:34:19,634 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/43941c6e16b14074889b575104ba8bb8 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/43941c6e16b14074889b575104ba8bb8 2024-11-14T21:34:19,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742325_1501 (size=12151) 2024-11-14T21:34:19,638 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/C of 81ffab6ef2cdc120372df3297622f922 into 43941c6e16b14074889b575104ba8bb8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:19,638 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:19,638 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/C, priority=12, startTime=1731620059005; duration=0sec 2024-11-14T21:34:19,638 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:19,638 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:C 2024-11-14T21:34:19,638 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/05efc3a2769a4c2e8eae37cd5139b932 2024-11-14T21:34:19,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/840c1c8a5dcb4c9184d70a2b0d5d0ed0 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/840c1c8a5dcb4c9184d70a2b0d5d0ed0 2024-11-14T21:34:19,655 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/840c1c8a5dcb4c9184d70a2b0d5d0ed0, entries=150, sequenceid=237, filesize=11.9 K 2024-11-14T21:34:19,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/e583a1d49ca7408c93dd22048ddbcd99 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/e583a1d49ca7408c93dd22048ddbcd99 2024-11-14T21:34:19,660 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/e583a1d49ca7408c93dd22048ddbcd99, entries=150, sequenceid=237, filesize=11.9 K 2024-11-14T21:34:19,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/05efc3a2769a4c2e8eae37cd5139b932 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/05efc3a2769a4c2e8eae37cd5139b932 2024-11-14T21:34:19,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:19,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620119660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:19,666 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/05efc3a2769a4c2e8eae37cd5139b932, entries=150, sequenceid=237, filesize=11.9 K 2024-11-14T21:34:19,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:19,669 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 81ffab6ef2cdc120372df3297622f922 in 552ms, sequenceid=237, compaction requested=false 2024-11-14T21:34:19,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:19,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:19,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-14T21:34:19,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:19,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T21:34:19,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:19,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:19,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:19,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:19,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:19,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:19,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-14T21:34:19,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-14T21:34:19,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6750 sec 2024-11-14T21:34:19,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.6940 sec 2024-11-14T21:34:19,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/c22b8c9e8a7547d09a9d778dc0894990 is 50, key is test_row_0/A:col10/1731620059311/Put/seqid=0 2024-11-14T21:34:19,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742326_1502 (size=14591) 2024-11-14T21:34:19,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:19,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620119703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:19,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:19,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620119703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:19,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:19,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620119704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:19,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620119704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:19,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:19,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620119831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:19,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:19,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620119831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:19,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:19,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:19,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620119831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:19,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:19,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:19,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620119831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:19,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:19,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:19,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620119971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:19,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:19,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:20,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620120041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:20,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620120041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:20,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:20,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:20,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620120043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:20,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:20,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620120044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:20,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/c22b8c9e8a7547d09a9d778dc0894990 2024-11-14T21:34:20,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/3da56c812438491ab4ba4a514ee9616d is 50, key is test_row_0/B:col10/1731620059311/Put/seqid=0 2024-11-14T21:34:20,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T21:34:20,141 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:34:20,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:34:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-14T21:34:20,148 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:34:20,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T21:34:20,153 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:34:20,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:34:20,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742327_1503 (size=12201) 2024-11-14T21:34:20,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/3da56c812438491ab4ba4a514ee9616d 2024-11-14T21:34:20,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/15072343f6954d9a9a5ef63826a6da6e is 50, key is test_row_0/C:col10/1731620059311/Put/seqid=0 2024-11-14T21:34:20,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742328_1504 (size=12201) 2024-11-14T21:34:20,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T21:34:20,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-14T21:34:20,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:20,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:20,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:20,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:20,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:20,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:20,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620120352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:20,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620120360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:20,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620120370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:20,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620120370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:20,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-14T21:34:20,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:20,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:20,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:20,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:20,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T21:34:20,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620120480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:20,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:20,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-14T21:34:20,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:20,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:20,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:20,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:20,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:20,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:20,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/15072343f6954d9a9a5ef63826a6da6e 2024-11-14T21:34:20,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/c22b8c9e8a7547d09a9d778dc0894990 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/c22b8c9e8a7547d09a9d778dc0894990 2024-11-14T21:34:20,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/c22b8c9e8a7547d09a9d778dc0894990, entries=200, sequenceid=260, filesize=14.2 K 2024-11-14T21:34:20,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/3da56c812438491ab4ba4a514ee9616d as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/3da56c812438491ab4ba4a514ee9616d 2024-11-14T21:34:20,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/3da56c812438491ab4ba4a514ee9616d, entries=150, sequenceid=260, filesize=11.9 K 2024-11-14T21:34:20,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/15072343f6954d9a9a5ef63826a6da6e as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/15072343f6954d9a9a5ef63826a6da6e 2024-11-14T21:34:20,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/15072343f6954d9a9a5ef63826a6da6e, entries=150, sequenceid=260, filesize=11.9 K 2024-11-14T21:34:20,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 81ffab6ef2cdc120372df3297622f922 in 1064ms, sequenceid=260, compaction requested=true 2024-11-14T21:34:20,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:20,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:34:20,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:20,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:34:20,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:34:20,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:34:20,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T21:34:20,734 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:20,734 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:20,739 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39439 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:20,739 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:20,740 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/C is initiating minor compaction (all files) 2024-11-14T21:34:20,740 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/A is initiating minor compaction (all files) 2024-11-14T21:34:20,740 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/C in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:20,740 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/A in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:20,740 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/1982b06e04064c6fab6405ee0babc2a2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/840c1c8a5dcb4c9184d70a2b0d5d0ed0, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/c22b8c9e8a7547d09a9d778dc0894990] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=38.5 K 2024-11-14T21:34:20,740 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/43941c6e16b14074889b575104ba8bb8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/05efc3a2769a4c2e8eae37cd5139b932, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/15072343f6954d9a9a5ef63826a6da6e] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=36.2 K 2024-11-14T21:34:20,740 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 1982b06e04064c6fab6405ee0babc2a2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731620056804 2024-11-14T21:34:20,741 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 840c1c8a5dcb4c9184d70a2b0d5d0ed0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731620058033 2024-11-14T21:34:20,741 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43941c6e16b14074889b575104ba8bb8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731620056804 2024-11-14T21:34:20,742 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting c22b8c9e8a7547d09a9d778dc0894990, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731620059299 2024-11-14T21:34:20,742 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05efc3a2769a4c2e8eae37cd5139b932, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731620058033 2024-11-14T21:34:20,742 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15072343f6954d9a9a5ef63826a6da6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731620059299 2024-11-14T21:34:20,769 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#C#compaction#431 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:20,770 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/8ef9e4f444c747cdb1cd8c0611655b4e is 50, key is test_row_0/C:col10/1731620059311/Put/seqid=0 2024-11-14T21:34:20,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-14T21:34:20,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:20,773 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T21:34:20,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:20,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:20,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:20,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:20,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:20,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:20,777 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#A#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:20,777 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/453ff2dfe5074590898ec1d5b1f07c7b is 50, key is test_row_0/A:col10/1731620059311/Put/seqid=0 2024-11-14T21:34:20,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T21:34:20,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/5d4c20a9aa5d463cb4b9653f4b1f8bee is 50, key is test_row_0/A:col10/1731620059702/Put/seqid=0 2024-11-14T21:34:20,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742329_1505 (size=12849) 2024-11-14T21:34:20,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742330_1506 (size=12849) 2024-11-14T21:34:20,809 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/453ff2dfe5074590898ec1d5b1f07c7b as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/453ff2dfe5074590898ec1d5b1f07c7b 2024-11-14T21:34:20,812 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/A of 81ffab6ef2cdc120372df3297622f922 into 453ff2dfe5074590898ec1d5b1f07c7b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:20,812 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:20,812 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/A, priority=13, startTime=1731620060733; duration=0sec 2024-11-14T21:34:20,812 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:20,812 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:A 2024-11-14T21:34:20,813 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:20,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742331_1507 (size=12301) 2024-11-14T21:34:20,825 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:20,825 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/B is initiating minor compaction (all files) 2024-11-14T21:34:20,825 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/B in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:20,825 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/097fbac7b96943738c819d5c90ce7d12, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/e583a1d49ca7408c93dd22048ddbcd99, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/3da56c812438491ab4ba4a514ee9616d] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=36.2 K 2024-11-14T21:34:20,825 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 097fbac7b96943738c819d5c90ce7d12, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731620056804 2024-11-14T21:34:20,826 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting e583a1d49ca7408c93dd22048ddbcd99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731620058033 2024-11-14T21:34:20,826 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 3da56c812438491ab4ba4a514ee9616d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731620059299 2024-11-14T21:34:20,835 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#B#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:20,836 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/7f36536e92b24b999f39d01847276f0e is 50, key is test_row_0/B:col10/1731620059311/Put/seqid=0 2024-11-14T21:34:20,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742332_1508 (size=12849) 2024-11-14T21:34:20,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:20,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:20,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620120939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:20,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:20,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620120939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:20,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620120955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:20,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620120956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:21,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620121060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:21,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620121060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:21,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:21,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:21,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620121080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:21,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:21,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620121081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:21,203 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/8ef9e4f444c747cdb1cd8c0611655b4e as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/8ef9e4f444c747cdb1cd8c0611655b4e 2024-11-14T21:34:21,208 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/C of 81ffab6ef2cdc120372df3297622f922 into 8ef9e4f444c747cdb1cd8c0611655b4e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:21,208 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:21,208 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/C, priority=13, startTime=1731620060733; duration=0sec 2024-11-14T21:34:21,208 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:21,208 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:C 2024-11-14T21:34:21,221 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/5d4c20a9aa5d463cb4b9653f4b1f8bee 2024-11-14T21:34:21,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/a31344306fc34abfb39420e210012467 is 50, key is test_row_0/B:col10/1731620059702/Put/seqid=0 2024-11-14T21:34:21,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742333_1509 (size=12301) 2024-11-14T21:34:21,262 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/a31344306fc34abfb39420e210012467 2024-11-14T21:34:21,285 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/7f36536e92b24b999f39d01847276f0e as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/7f36536e92b24b999f39d01847276f0e 2024-11-14T21:34:21,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/30fd86931f5149f593fd06e5000d61b1 is 50, key is test_row_0/C:col10/1731620059702/Put/seqid=0 2024-11-14T21:34:21,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T21:34:21,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:21,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620121281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:21,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:21,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620121281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:21,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:21,308 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/B of 81ffab6ef2cdc120372df3297622f922 into 7f36536e92b24b999f39d01847276f0e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:21,308 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:21,308 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/B, priority=13, startTime=1731620060733; duration=0sec 2024-11-14T21:34:21,308 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:21,308 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:B 2024-11-14T21:34:21,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:21,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620121303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:21,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:21,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:21,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620121306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:21,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:21,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742334_1510 (size=12301) 2024-11-14T21:34:21,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:21,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620121503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:21,515 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_0' in 81ffab6ef2cdc120372df3297622f922 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:34:21,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:21,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620121601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:21,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620121602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:21,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620121630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:21,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:21,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620121630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:21,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:21,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:21,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:21,727 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/30fd86931f5149f593fd06e5000d61b1 2024-11-14T21:34:21,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/5d4c20a9aa5d463cb4b9653f4b1f8bee as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/5d4c20a9aa5d463cb4b9653f4b1f8bee 2024-11-14T21:34:21,737 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/5d4c20a9aa5d463cb4b9653f4b1f8bee, entries=150, sequenceid=273, filesize=12.0 K 2024-11-14T21:34:21,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/a31344306fc34abfb39420e210012467 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/a31344306fc34abfb39420e210012467 2024-11-14T21:34:21,754 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/a31344306fc34abfb39420e210012467, entries=150, sequenceid=273, filesize=12.0 K 2024-11-14T21:34:21,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/30fd86931f5149f593fd06e5000d61b1 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/30fd86931f5149f593fd06e5000d61b1 2024-11-14T21:34:21,769 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/30fd86931f5149f593fd06e5000d61b1, entries=150, sequenceid=273, filesize=12.0 K 2024-11-14T21:34:21,770 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 81ffab6ef2cdc120372df3297622f922 in 997ms, sequenceid=273, compaction requested=false 2024-11-14T21:34:21,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:21,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:21,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-14T21:34:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-14T21:34:21,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-14T21:34:21,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6260 sec 2024-11-14T21:34:21,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.6440 sec 2024-11-14T21:34:22,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T21:34:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:22,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731620060935, max timestamp=1731620062132Num uniques 859; 2024-11-14T21:34:22,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T21:34:22,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731620060935, max timestamp=1731620062132Num uniques 859; 2024-11-14T21:34:22,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T21:34:22,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731620060950, max timestamp=1731620062132Num uniques 859; 2024-11-14T21:34:22,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:34:22,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:22,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:22,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:22,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:22,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:22,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:22,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/1f39814d231b462fae551e4ab1e7641e is 50, key is test_row_0/A:col10/1731620062127/Put/seqid=0 2024-11-14T21:34:22,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742335_1511 (size=12301) 2024-11-14T21:34:22,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731620062211, max timestamp=1731620062260Num uniques 859; 2024-11-14T21:34:22,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731620062211, max timestamp=1731620062260Num uniques 859; 2024-11-14T21:34:22,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:34:22,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:34:22,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731620062240, max timestamp=1731620062260Num uniques 859; 2024-11-14T21:34:22,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:34:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T21:34:22,301 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:34:22,302 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:34:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-14T21:34:22,304 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:34:22,305 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:34:22,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:34:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T21:34:22,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731620062273, max timestamp=1731620062358Num uniques 1118; 2024-11-14T21:34:22,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312  suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-11-14T21:34:22,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731620062273, max timestamp=1731620062358Num uniques 1118; 2024-11-14T21:34:22,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312  suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-11-14T21:34:22,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731620062273, max timestamp=1731620062358Num uniques 1018; 2024-11-14T21:34:22,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-14T21:34:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T21:34:22,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620122440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:22,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731620062412, max timestamp=1731620062451Num uniques 1177; 2024-11-14T21:34:22,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-11-14T21:34:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:22,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T21:34:22,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:22,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:22,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:22,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:22,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731620062412, max timestamp=1731620062451Num uniques 1177; 2024-11-14T21:34:22,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-11-14T21:34:22,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:22,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731620062412, max timestamp=1731620062451Num uniques 1177; 2024-11-14T21:34:22,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-11-14T21:34:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:22,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731620062540, max timestamp=1731620062588Num uniques 1086; 2024-11-14T21:34:22,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-14T21:34:22,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731620062540, max timestamp=1731620062588Num uniques 1086; 2024-11-14T21:34:22,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-14T21:34:22,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731620062540, max timestamp=1731620062588Num uniques 1086; 2024-11-14T21:34:22,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-14T21:34:22,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/1f39814d231b462fae551e4ab1e7641e 2024-11-14T21:34:22,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T21:34:22,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:22,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:22,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:22,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:22,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T21:34:22,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/e41a4615be6347d0a5413276a19df10c is 50, key is test_row_0/B:col10/1731620062127/Put/seqid=0 2024-11-14T21:34:22,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742336_1512 (size=12301) 2024-11-14T21:34:22,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/e41a4615be6347d0a5413276a19df10c 2024-11-14T21:34:22,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731620062630, max timestamp=1731620062683Num uniques 1095; 2024-11-14T21:34:22,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920  suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-14T21:34:22,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731620062630, max timestamp=1731620062683Num uniques 1045; 2024-11-14T21:34:22,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920  suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-14T21:34:22,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731620062630, max timestamp=1731620062683Num uniques 1045; 2024-11-14T21:34:22,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520  suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-14T21:34:22,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/fc880f45bbe1472591584df1a2724179 is 50, key is test_row_0/C:col10/1731620062127/Put/seqid=0 2024-11-14T21:34:22,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742337_1513 (size=12301) 2024-11-14T21:34:22,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/fc880f45bbe1472591584df1a2724179 2024-11-14T21:34:22,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/1f39814d231b462fae551e4ab1e7641e as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/1f39814d231b462fae551e4ab1e7641e 2024-11-14T21:34:22,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T21:34:22,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:22,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:22,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:22,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:22,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:22,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/1f39814d231b462fae551e4ab1e7641e, entries=150, sequenceid=297, filesize=12.0 K 2024-11-14T21:34:22,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/e41a4615be6347d0a5413276a19df10c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/e41a4615be6347d0a5413276a19df10c 2024-11-14T21:34:22,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/e41a4615be6347d0a5413276a19df10c, entries=150, sequenceid=297, filesize=12.0 K 2024-11-14T21:34:22,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/fc880f45bbe1472591584df1a2724179 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/fc880f45bbe1472591584df1a2724179 2024-11-14T21:34:22,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620122779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:22,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731620062719, max timestamp=1731620062812Num uniques 1154; 2024-11-14T21:34:22,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=9340, suffix heap size=147984,new segment heap size=21456  suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-11-14T21:34:22,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620122789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:22,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731620062719, max timestamp=1731620062812Num uniques 1154; 2024-11-14T21:34:22,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=9340, suffix heap size=147984,new segment heap size=21456  suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-11-14T21:34:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:22,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731620062719, max timestamp=1731620062812Num uniques 1054; 2024-11-14T21:34:22,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/fc880f45bbe1472591584df1a2724179, entries=150, sequenceid=297, filesize=12.0 K 2024-11-14T21:34:22,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456  suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-14T21:34:22,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~31.31 KB/32060, heapSize ~76.34 KB/78176, currentSize=33.54 KB/34350 for 81ffab6ef2cdc120372df3297622f922 in 689ms, sequenceid=297, compaction requested=true 2024-11-14T21:34:22,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:22,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:34:22,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:22,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:34:22,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:34:22,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:34:22,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T21:34:22,826 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:22,826 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:22,827 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:22,827 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/A is initiating minor compaction (all files) 2024-11-14T21:34:22,827 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/A in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:22,827 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/453ff2dfe5074590898ec1d5b1f07c7b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/5d4c20a9aa5d463cb4b9653f4b1f8bee, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/1f39814d231b462fae551e4ab1e7641e] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=36.6 K 2024-11-14T21:34:22,830 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 453ff2dfe5074590898ec1d5b1f07c7b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731620059299 2024-11-14T21:34:22,831 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d4c20a9aa5d463cb4b9653f4b1f8bee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731620059700 2024-11-14T21:34:22,831 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f39814d231b462fae551e4ab1e7641e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731620060952 2024-11-14T21:34:22,841 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:22,841 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/C is initiating minor compaction (all files) 2024-11-14T21:34:22,841 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/C in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:22,842 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/8ef9e4f444c747cdb1cd8c0611655b4e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/30fd86931f5149f593fd06e5000d61b1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/fc880f45bbe1472591584df1a2724179] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=36.6 K 2024-11-14T21:34:22,842 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ef9e4f444c747cdb1cd8c0611655b4e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731620059299 2024-11-14T21:34:22,850 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30fd86931f5149f593fd06e5000d61b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731620059700 2024-11-14T21:34:22,854 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc880f45bbe1472591584df1a2724179, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731620060952 2024-11-14T21:34:22,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=53.67 KB heapSize=141.70 KB 2024-11-14T21:34:22,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:22,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:22,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:22,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:22,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:22,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:22,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/d4652d0351414d2cbb826abfaef7f0a5 is 50, key is test_row_0/A:col10/1731620062859/Put/seqid=0 2024-11-14T21:34:22,883 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#A#compaction#441 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:22,883 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/8134793ea7b540e0bdbf8eb29a5317f3 is 50, key is test_row_0/A:col10/1731620062127/Put/seqid=0 2024-11-14T21:34:22,919 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#C#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:22,919 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/c7103aca964b4610aed2631d6d7b5842 is 50, key is test_row_0/C:col10/1731620062127/Put/seqid=0 2024-11-14T21:34:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T21:34:22,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742338_1514 (size=14741) 2024-11-14T21:34:22,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/d4652d0351414d2cbb826abfaef7f0a5 2024-11-14T21:34:22,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T21:34:22,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:22,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:22,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:22,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:22,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:22,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/833e87a0eef04eccb02215628b8616d8 is 50, key is test_row_0/B:col10/1731620062859/Put/seqid=0 2024-11-14T21:34:23,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742340_1516 (size=13051) 2024-11-14T21:34:23,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742339_1515 (size=13051) 2024-11-14T21:34:23,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731620062979, max timestamp=1731620063019Num uniques 859; 2024-11-14T21:34:23,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:34:23,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731620062979, max timestamp=1731620063019Num uniques 859; 2024-11-14T21:34:23,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:34:23,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731620062979, max timestamp=1731620063019Num uniques 859; 2024-11-14T21:34:23,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:34:23,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742341_1517 (size=12301) 2024-11-14T21:34:23,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/833e87a0eef04eccb02215628b8616d8 2024-11-14T21:34:23,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/0567e5ce9dcd413dbb9b2544830d7f4e is 50, key is test_row_0/C:col10/1731620062859/Put/seqid=0 2024-11-14T21:34:23,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742342_1518 (size=12301) 2024-11-14T21:34:23,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/0567e5ce9dcd413dbb9b2544830d7f4e 2024-11-14T21:34:23,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/d4652d0351414d2cbb826abfaef7f0a5 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/d4652d0351414d2cbb826abfaef7f0a5 2024-11-14T21:34:23,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T21:34:23,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:23,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:23,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:23,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:23,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:23,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/d4652d0351414d2cbb826abfaef7f0a5, entries=200, sequenceid=407, filesize=14.4 K 2024-11-14T21:34:23,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/833e87a0eef04eccb02215628b8616d8 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/833e87a0eef04eccb02215628b8616d8 2024-11-14T21:34:23,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620123148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:23,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:23,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620123153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:23,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:23,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:23,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620123170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:23,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:23,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/833e87a0eef04eccb02215628b8616d8, entries=150, sequenceid=407, filesize=12.0 K 2024-11-14T21:34:23,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/0567e5ce9dcd413dbb9b2544830d7f4e as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/0567e5ce9dcd413dbb9b2544830d7f4e 2024-11-14T21:34:23,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/0567e5ce9dcd413dbb9b2544830d7f4e, entries=150, sequenceid=407, filesize=12.0 K 2024-11-14T21:34:23,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.66 KB/145056, currentSize=134.18 KB/137400 for 81ffab6ef2cdc120372df3297622f922 in 355ms, sequenceid=407, compaction requested=true 2024-11-14T21:34:23,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:23,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:A, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:34:23,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:34:23,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:B, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:34:23,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T21:34:23,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:34:23,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-14T21:34:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:23,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-14T21:34:23,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:23,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:23,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:23,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:23,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:23,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:23,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/2bb47ca28a914626ae816fa75120a93f is 50, key is test_row_0/A:col10/1731620063181/Put/seqid=0 2024-11-14T21:34:23,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742343_1519 (size=14741) 2024-11-14T21:34:23,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T21:34:23,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:23,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:23,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:23,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:23,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:23,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620123329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:23,338 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:23,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620123329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:23,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620123341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:23,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:23,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620123346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:23,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:23,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:23,418 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/c7103aca964b4610aed2631d6d7b5842 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/c7103aca964b4610aed2631d6d7b5842 2024-11-14T21:34:23,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T21:34:23,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:23,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:23,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:23,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:23,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:23,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:23,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T21:34:23,446 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/C of 81ffab6ef2cdc120372df3297622f922 into c7103aca964b4610aed2631d6d7b5842(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:23,446 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:23,446 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/C, priority=13, startTime=1731620062826; duration=0sec 2024-11-14T21:34:23,446 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-14T21:34:23,446 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:C 2024-11-14T21:34:23,446 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-14T21:34:23,451 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/8134793ea7b540e0bdbf8eb29a5317f3 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/8134793ea7b540e0bdbf8eb29a5317f3 2024-11-14T21:34:23,458 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T21:34:23,458 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T21:34:23,458 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. because compaction request was cancelled 2024-11-14T21:34:23,459 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:A 2024-11-14T21:34:23,459 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:C 2024-11-14T21:34:23,459 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:34:23,460 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49752 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:34:23,460 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/B is initiating minor compaction (all files) 2024-11-14T21:34:23,460 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/B in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:23,460 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/7f36536e92b24b999f39d01847276f0e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/a31344306fc34abfb39420e210012467, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/e41a4615be6347d0a5413276a19df10c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/833e87a0eef04eccb02215628b8616d8] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=48.6 K 2024-11-14T21:34:23,460 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f36536e92b24b999f39d01847276f0e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731620059299 2024-11-14T21:34:23,461 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting a31344306fc34abfb39420e210012467, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731620059700 2024-11-14T21:34:23,461 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting e41a4615be6347d0a5413276a19df10c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731620060952 2024-11-14T21:34:23,462 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 833e87a0eef04eccb02215628b8616d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731620062812 2024-11-14T21:34:23,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620123453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:23,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:23,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620123462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:23,482 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/A of 81ffab6ef2cdc120372df3297622f922 into 8134793ea7b540e0bdbf8eb29a5317f3(size=12.7 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:23,482 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:23,483 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/A, priority=13, startTime=1731620062826; duration=0sec 2024-11-14T21:34:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:23,483 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:34:23,483 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:A 2024-11-14T21:34:23,483 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-14T21:34:23,483 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T21:34:23,483 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T21:34:23,483 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. because compaction request was cancelled 2024-11-14T21:34:23,483 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:B 2024-11-14T21:34:23,483 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-14T21:34:23,483 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T21:34:23,484 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T21:34:23,484 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. because compaction request was cancelled 2024-11-14T21:34:23,484 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:C 2024-11-14T21:34:23,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620123481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:23,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:23,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620123481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:23,493 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#B#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:23,493 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/b6b8807cd26149f0a446af2465f2ae6a is 50, key is test_row_0/B:col10/1731620062859/Put/seqid=0 2024-11-14T21:34:23,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742344_1520 (size=13085) 2024-11-14T21:34:23,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620123523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:23,538 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_0' in 81ffab6ef2cdc120372df3297622f922 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:34:23,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:23,559 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/b6b8807cd26149f0a446af2465f2ae6a as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/b6b8807cd26149f0a446af2465f2ae6a 2024-11-14T21:34:23,583 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/B of 81ffab6ef2cdc120372df3297622f922 into b6b8807cd26149f0a446af2465f2ae6a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:23,583 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:23,583 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/B, priority=12, startTime=1731620063215; duration=0sec 2024-11-14T21:34:23,583 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:23,583 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:B 2024-11-14T21:34:23,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T21:34:23,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:23,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:23,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:23,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:23,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:23,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:23,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/2bb47ca28a914626ae816fa75120a93f 2024-11-14T21:34:23,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/ada4d91c69ab4800abbb84293e7f6b6c is 50, key is test_row_0/B:col10/1731620063181/Put/seqid=0 2024-11-14T21:34:23,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620123671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:23,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:23,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:23,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620123690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:23,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:23,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620123706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:23,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620123706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:23,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742345_1521 (size=12301) 2024-11-14T21:34:23,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T21:34:23,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:23,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:23,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:23,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:23,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:23,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T21:34:23,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:23,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:23,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:23,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:23,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:23,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620123990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:24,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620124001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:24,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:24,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:24,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:24,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620124021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:24,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:24,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620124021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:24,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:24,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:24,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:24,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:24,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:24,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T21:34:24,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:24,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:24,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:24,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:24,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:24,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/ada4d91c69ab4800abbb84293e7f6b6c 2024-11-14T21:34:24,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/e98cf794dcf64f60a72113dd3b91b0ed is 50, key is test_row_0/C:col10/1731620063181/Put/seqid=0 2024-11-14T21:34:24,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742346_1522 (size=12301) 2024-11-14T21:34:24,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/e98cf794dcf64f60a72113dd3b91b0ed 2024-11-14T21:34:24,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/2bb47ca28a914626ae816fa75120a93f as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/2bb47ca28a914626ae816fa75120a93f 2024-11-14T21:34:24,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/2bb47ca28a914626ae816fa75120a93f, entries=200, sequenceid=443, filesize=14.4 K 2024-11-14T21:34:24,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/ada4d91c69ab4800abbb84293e7f6b6c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/ada4d91c69ab4800abbb84293e7f6b6c 2024-11-14T21:34:24,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T21:34:24,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:24,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:24,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:24,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:24,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:24,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/ada4d91c69ab4800abbb84293e7f6b6c, entries=150, sequenceid=443, filesize=12.0 K 2024-11-14T21:34:24,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/e98cf794dcf64f60a72113dd3b91b0ed as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/e98cf794dcf64f60a72113dd3b91b0ed 2024-11-14T21:34:24,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/e98cf794dcf64f60a72113dd3b91b0ed, entries=150, sequenceid=443, filesize=12.0 K 2024-11-14T21:34:24,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=46.96 KB/48090 for 81ffab6ef2cdc120372df3297622f922 in 1040ms, sequenceid=443, compaction requested=true 2024-11-14T21:34:24,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:24,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:34:24,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:24,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:34:24,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:34:24,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:34:24,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T21:34:24,262 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:24,262 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:24,263 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:24,263 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/A is initiating minor compaction (all files) 2024-11-14T21:34:24,263 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/A in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:24,263 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/8134793ea7b540e0bdbf8eb29a5317f3, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/d4652d0351414d2cbb826abfaef7f0a5, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/2bb47ca28a914626ae816fa75120a93f] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=41.5 K 2024-11-14T21:34:24,264 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:24,264 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8134793ea7b540e0bdbf8eb29a5317f3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731620060952 2024-11-14T21:34:24,264 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/C is initiating minor compaction (all files) 2024-11-14T21:34:24,264 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/C in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:24,264 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/c7103aca964b4610aed2631d6d7b5842, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/0567e5ce9dcd413dbb9b2544830d7f4e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/e98cf794dcf64f60a72113dd3b91b0ed] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=36.8 K 2024-11-14T21:34:24,264 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4652d0351414d2cbb826abfaef7f0a5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731620062812 2024-11-14T21:34:24,264 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting c7103aca964b4610aed2631d6d7b5842, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731620060952 2024-11-14T21:34:24,264 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bb47ca28a914626ae816fa75120a93f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731620063133 2024-11-14T21:34:24,264 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 0567e5ce9dcd413dbb9b2544830d7f4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731620062812 2024-11-14T21:34:24,265 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting e98cf794dcf64f60a72113dd3b91b0ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731620063133 2024-11-14T21:34:24,286 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#A#compaction#449 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:24,287 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/9dcee99a8b9a4257b51e29a1290f74c7 is 50, key is test_row_0/A:col10/1731620063181/Put/seqid=0 2024-11-14T21:34:24,302 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#C#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:24,303 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/1c2c63a7bb814884986fade17eb67dca is 50, key is test_row_0/C:col10/1731620063181/Put/seqid=0 2024-11-14T21:34:24,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742347_1523 (size=13153) 2024-11-14T21:34:24,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742348_1524 (size=13153) 2024-11-14T21:34:24,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T21:34:24,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:24,394 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T21:34:24,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:24,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:24,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:24,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:24,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:24,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:24,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/c6b19da5b26f42a0a1fff50b532fc0f0 is 50, key is test_row_0/A:col10/1731620063330/Put/seqid=0 2024-11-14T21:34:24,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742349_1525 (size=12301) 2024-11-14T21:34:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T21:34:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:24,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:24,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731620064555, max timestamp=1731620064614Num uniques 859; 2024-11-14T21:34:24,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:34:24,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731620064555, max timestamp=1731620064614Num uniques 859; 2024-11-14T21:34:24,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:34:24,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731620064555, max timestamp=1731620064614Num uniques 859; 2024-11-14T21:34:24,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:34:24,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620124678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:24,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:24,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620124690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:24,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620124695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:24,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:24,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:24,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:24,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620124696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:24,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:24,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:24,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:24,748 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/9dcee99a8b9a4257b51e29a1290f74c7 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/9dcee99a8b9a4257b51e29a1290f74c7 2024-11-14T21:34:24,766 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/A of 81ffab6ef2cdc120372df3297622f922 into 9dcee99a8b9a4257b51e29a1290f74c7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:24,767 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:24,767 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/A, priority=13, startTime=1731620064261; duration=0sec 2024-11-14T21:34:24,767 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:24,767 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:A 2024-11-14T21:34:24,767 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-14T21:34:24,772 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T21:34:24,772 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T21:34:24,772 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. because compaction request was cancelled 2024-11-14T21:34:24,772 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:B 2024-11-14T21:34:24,775 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/1c2c63a7bb814884986fade17eb67dca as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/1c2c63a7bb814884986fade17eb67dca 2024-11-14T21:34:24,781 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/C of 81ffab6ef2cdc120372df3297622f922 into 1c2c63a7bb814884986fade17eb67dca(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:24,781 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:24,781 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/C, priority=13, startTime=1731620064261; duration=0sec 2024-11-14T21:34:24,781 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:24,781 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:C 2024-11-14T21:34:24,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620124802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:24,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620124822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:24,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620124823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:24,829 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/c6b19da5b26f42a0a1fff50b532fc0f0 2024-11-14T21:34:24,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620124823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:24,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/5682d168eda0446c82b83bd33eba7275 is 50, key is test_row_0/B:col10/1731620063330/Put/seqid=0 2024-11-14T21:34:24,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742350_1526 (size=12301) 2024-11-14T21:34:24,860 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/5682d168eda0446c82b83bd33eba7275 2024-11-14T21:34:24,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/4cb2d335e82b4b968f5852bade34559f is 50, key is test_row_0/C:col10/1731620063330/Put/seqid=0 2024-11-14T21:34:24,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742351_1527 (size=12301) 2024-11-14T21:34:25,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620125028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:25,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620125043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:25,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620125043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:25,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620125046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:25,283 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/4cb2d335e82b4b968f5852bade34559f 2024-11-14T21:34:25,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/c6b19da5b26f42a0a1fff50b532fc0f0 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/c6b19da5b26f42a0a1fff50b532fc0f0 2024-11-14T21:34:25,298 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/c6b19da5b26f42a0a1fff50b532fc0f0, entries=150, sequenceid=456, filesize=12.0 K 2024-11-14T21:34:25,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/5682d168eda0446c82b83bd33eba7275 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/5682d168eda0446c82b83bd33eba7275 2024-11-14T21:34:25,302 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/5682d168eda0446c82b83bd33eba7275, entries=150, sequenceid=456, filesize=12.0 K 2024-11-14T21:34:25,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/4cb2d335e82b4b968f5852bade34559f as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/4cb2d335e82b4b968f5852bade34559f 2024-11-14T21:34:25,306 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/4cb2d335e82b4b968f5852bade34559f, entries=150, sequenceid=456, filesize=12.0 K 2024-11-14T21:34:25,315 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 81ffab6ef2cdc120372df3297622f922 in 922ms, sequenceid=456, compaction requested=true 2024-11-14T21:34:25,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:25,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:25,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-14T21:34:25,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-14T21:34:25,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-14T21:34:25,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0120 sec 2024-11-14T21:34:25,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 3.0170 sec 2024-11-14T21:34:25,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-14T21:34:25,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:25,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:25,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:25,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:25,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:25,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:25,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/905897a2e5c542a2bb1d24ba9d8331b2 is 50, key is test_row_0/A:col10/1731620065365/Put/seqid=0 2024-11-14T21:34:25,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742352_1528 (size=14741) 2024-11-14T21:34:25,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/905897a2e5c542a2bb1d24ba9d8331b2 2024-11-14T21:34:25,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620125394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:25,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620125395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:25,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:25,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620125428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:25,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620125429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:25,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:25,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/2b54376b01034f718ea69a9b3bb134ff is 50, key is test_row_0/B:col10/1731620065365/Put/seqid=0 2024-11-14T21:34:25,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742353_1529 (size=12301) 2024-11-14T21:34:25,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/2b54376b01034f718ea69a9b3bb134ff 2024-11-14T21:34:25,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/670d4880998a42e2999fef72c66a2800 is 50, key is test_row_0/C:col10/1731620065365/Put/seqid=0 2024-11-14T21:34:25,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620125540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:25,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620125540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:25,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742354_1530 (size=12301) 2024-11-14T21:34:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:25,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/670d4880998a42e2999fef72c66a2800 2024-11-14T21:34:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:25,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620125563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620125563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:25,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/905897a2e5c542a2bb1d24ba9d8331b2 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/905897a2e5c542a2bb1d24ba9d8331b2 2024-11-14T21:34:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:25,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/905897a2e5c542a2bb1d24ba9d8331b2, entries=200, sequenceid=494, filesize=14.4 K 2024-11-14T21:34:25,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/2b54376b01034f718ea69a9b3bb134ff as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/2b54376b01034f718ea69a9b3bb134ff 2024-11-14T21:34:25,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/2b54376b01034f718ea69a9b3bb134ff, entries=150, sequenceid=494, filesize=12.0 K 2024-11-14T21:34:25,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/670d4880998a42e2999fef72c66a2800 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/670d4880998a42e2999fef72c66a2800 2024-11-14T21:34:25,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/670d4880998a42e2999fef72c66a2800, entries=150, sequenceid=494, filesize=12.0 K 2024-11-14T21:34:25,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 81ffab6ef2cdc120372df3297622f922 in 282ms, sequenceid=494, compaction requested=true 2024-11-14T21:34:25,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:25,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:34:25,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:25,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:34:25,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:34:25,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:34:25,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T21:34:25,650 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:34:25,652 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:25,656 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:34:25,656 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/B is initiating minor compaction (all files) 2024-11-14T21:34:25,657 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/B in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:25,657 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/b6b8807cd26149f0a446af2465f2ae6a, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/ada4d91c69ab4800abbb84293e7f6b6c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/5682d168eda0446c82b83bd33eba7275, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/2b54376b01034f718ea69a9b3bb134ff] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=48.8 K 2024-11-14T21:34:25,657 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:25,657 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/C is initiating minor compaction (all files) 2024-11-14T21:34:25,657 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/C in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:25,657 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/1c2c63a7bb814884986fade17eb67dca, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/4cb2d335e82b4b968f5852bade34559f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/670d4880998a42e2999fef72c66a2800] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=36.9 K 2024-11-14T21:34:25,657 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c2c63a7bb814884986fade17eb67dca, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731620063133 2024-11-14T21:34:25,657 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6b8807cd26149f0a446af2465f2ae6a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731620062812 2024-11-14T21:34:25,658 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cb2d335e82b4b968f5852bade34559f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731620063327 2024-11-14T21:34:25,658 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 670d4880998a42e2999fef72c66a2800, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731620064646 2024-11-14T21:34:25,658 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting ada4d91c69ab4800abbb84293e7f6b6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731620063133 2024-11-14T21:34:25,661 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5682d168eda0446c82b83bd33eba7275, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731620063327 2024-11-14T21:34:25,661 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b54376b01034f718ea69a9b3bb134ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731620064646 2024-11-14T21:34:25,685 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#C#compaction#457 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:25,685 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/3036f24da4a5444eba558ef66f0e2de9 is 50, key is test_row_0/C:col10/1731620065365/Put/seqid=0 2024-11-14T21:34:25,689 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#B#compaction#458 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:25,689 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/fe79f2bf0b544671b7433de2dd45d72b is 50, key is test_row_0/B:col10/1731620065365/Put/seqid=0 2024-11-14T21:34:25,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742355_1531 (size=13255) 2024-11-14T21:34:25,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742356_1532 (size=13221) 2024-11-14T21:34:25,782 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/3036f24da4a5444eba558ef66f0e2de9 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/3036f24da4a5444eba558ef66f0e2de9 2024-11-14T21:34:25,797 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/fe79f2bf0b544671b7433de2dd45d72b as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/fe79f2bf0b544671b7433de2dd45d72b 2024-11-14T21:34:25,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T21:34:25,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:25,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:25,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:25,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:25,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:25,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:25,809 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/C of 81ffab6ef2cdc120372df3297622f922 into 3036f24da4a5444eba558ef66f0e2de9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:25,809 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:25,809 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/C, priority=13, startTime=1731620065650; duration=0sec 2024-11-14T21:34:25,809 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:25,809 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:C 2024-11-14T21:34:25,809 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:25,824 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:25,825 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/A is initiating minor compaction (all files) 2024-11-14T21:34:25,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/5a1d9822aafe4d998722e07e9326fbb5 is 50, key is test_row_0/A:col10/1731620065786/Put/seqid=0 2024-11-14T21:34:25,825 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/A in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:25,825 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/9dcee99a8b9a4257b51e29a1290f74c7, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/c6b19da5b26f42a0a1fff50b532fc0f0, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/905897a2e5c542a2bb1d24ba9d8331b2] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=39.3 K 2024-11-14T21:34:25,830 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dcee99a8b9a4257b51e29a1290f74c7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731620063133 2024-11-14T21:34:25,830 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/B of 81ffab6ef2cdc120372df3297622f922 into fe79f2bf0b544671b7433de2dd45d72b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:25,830 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:25,830 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/B, priority=12, startTime=1731620065650; duration=0sec 2024-11-14T21:34:25,830 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:25,830 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:B 2024-11-14T21:34:25,831 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting c6b19da5b26f42a0a1fff50b532fc0f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731620063327 2024-11-14T21:34:25,831 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 905897a2e5c542a2bb1d24ba9d8331b2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731620064646 2024-11-14T21:34:25,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742357_1533 (size=17181) 2024-11-14T21:34:25,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/5a1d9822aafe4d998722e07e9326fbb5 2024-11-14T21:34:25,857 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#A#compaction#460 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:25,858 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/1e055a85a0c34f1290b836b8414c8899 is 50, key is test_row_0/A:col10/1731620065365/Put/seqid=0 2024-11-14T21:34:25,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/02fb5ac2486c4fe5aad9037586eec6ec is 50, key is test_row_0/B:col10/1731620065786/Put/seqid=0 2024-11-14T21:34:25,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742358_1534 (size=13255) 2024-11-14T21:34:25,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742359_1535 (size=12301) 2024-11-14T21:34:25,910 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731620065849, max timestamp=1731620065907Num uniques 859; 2024-11-14T21:34:25,910 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:34:25,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731620065849, max timestamp=1731620065907Num uniques 859; 2024-11-14T21:34:25,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:34:25,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731620065849, max timestamp=1731620065907Num uniques 859; 2024-11-14T21:34:25,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:34:25,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620125963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:25,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:25,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620125975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:26,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:26,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620125989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:26,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620125992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:26,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620126102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:26,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620126102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:26,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:26,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:26,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620126126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:26,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620126132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:26,282 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/1e055a85a0c34f1290b836b8414c8899 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/1e055a85a0c34f1290b836b8414c8899 2024-11-14T21:34:26,292 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/A of 81ffab6ef2cdc120372df3297622f922 into 1e055a85a0c34f1290b836b8414c8899(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:26,292 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:26,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/02fb5ac2486c4fe5aad9037586eec6ec 2024-11-14T21:34:26,292 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/A, priority=13, startTime=1731620065650; duration=0sec 2024-11-14T21:34:26,292 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:26,292 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:A 2024-11-14T21:34:26,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/14ecfb30764342ffb8e47ef4e8a4c9a2 is 50, key is test_row_0/C:col10/1731620065786/Put/seqid=0 2024-11-14T21:34:26,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742360_1536 (size=12301) 2024-11-14T21:34:26,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/14ecfb30764342ffb8e47ef4e8a4c9a2 2024-11-14T21:34:26,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/5a1d9822aafe4d998722e07e9326fbb5 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/5a1d9822aafe4d998722e07e9326fbb5 2024-11-14T21:34:26,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/5a1d9822aafe4d998722e07e9326fbb5, entries=250, sequenceid=505, filesize=16.8 K 2024-11-14T21:34:26,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/02fb5ac2486c4fe5aad9037586eec6ec as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/02fb5ac2486c4fe5aad9037586eec6ec 2024-11-14T21:34:26,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/02fb5ac2486c4fe5aad9037586eec6ec, entries=150, sequenceid=505, filesize=12.0 K 2024-11-14T21:34:26,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/14ecfb30764342ffb8e47ef4e8a4c9a2 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/14ecfb30764342ffb8e47ef4e8a4c9a2 2024-11-14T21:34:26,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/14ecfb30764342ffb8e47ef4e8a4c9a2, entries=150, sequenceid=505, filesize=12.0 K 2024-11-14T21:34:26,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 81ffab6ef2cdc120372df3297622f922 in 538ms, sequenceid=505, compaction requested=false 2024-11-14T21:34:26,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:26,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-14T21:34:26,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:26,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:26,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:26,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:26,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:26,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:26,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/fc40a2160cb54c64813929059ab26a25 is 50, key is test_row_0/A:col10/1731620066346/Put/seqid=0 2024-11-14T21:34:26,379 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T21:34:26,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742361_1537 (size=12301) 2024-11-14T21:34:26,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620126370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:26,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620126391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:26,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:26,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620126392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:26,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620126392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:26,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T21:34:26,481 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:34:26,482 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:34:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-14T21:34:26,484 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:34:26,486 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:34:26,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:34:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T21:34:26,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620126501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:26,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:26,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620126520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:26,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:26,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620126520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:26,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:26,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620126532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T21:34:26,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T21:34:26,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:26,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:26,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:26,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:26,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:26,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620126720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:26,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:26,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620126740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:26,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620126741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:26,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:26,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620126752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:26,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/fc40a2160cb54c64813929059ab26a25 2024-11-14T21:34:26,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T21:34:26,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:26,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:26,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:26,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:26,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T21:34:26,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/c3974adda2f04db7ab82c1d28f6b7584 is 50, key is test_row_0/B:col10/1731620066346/Put/seqid=0 2024-11-14T21:34:26,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742362_1538 (size=12301) 2024-11-14T21:34:26,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T21:34:26,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:26,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:26,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:26,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:26,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:27,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620127041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:27,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:27,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:27,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:27,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620127061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:27,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620127061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:27,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620127061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:27,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T21:34:27,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:27,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:27,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:27,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:27,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T21:34:27,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T21:34:27,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:27,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:27,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:27,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:27,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:27,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/c3974adda2f04db7ab82c1d28f6b7584 2024-11-14T21:34:27,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/59e9019d20cf4f76b34cc3433869377d is 50, key is test_row_0/C:col10/1731620066346/Put/seqid=0 2024-11-14T21:34:27,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742363_1539 (size=12301) 2024-11-14T21:34:27,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T21:34:27,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:27,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:27,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:27,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:27,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:27,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:27,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1731620127561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:27,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T21:34:27,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:27,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:27,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:27,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:27,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:27,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1731620127581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:27,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35870 deadline: 1731620127581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:27,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:27,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:27,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:27,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:27,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:27,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35846 deadline: 1731620127581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:27,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:27,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:27,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:27,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:27,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731620127581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:27,593 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_0' in 81ffab6ef2cdc120372df3297622f922 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8280 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:34:27,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 , the old value is region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:27,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:27,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., hostname=ef788a4cade8,37683,1731619918560, seqNum=2 because the exception is null or not the one we care about 2024-11-14T21:34:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T21:34:27,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/59e9019d20cf4f76b34cc3433869377d 2024-11-14T21:34:27,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T21:34:27,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:27,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. as already flushing 2024-11-14T21:34:27,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:27,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:27,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:27,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/fc40a2160cb54c64813929059ab26a25 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/fc40a2160cb54c64813929059ab26a25 2024-11-14T21:34:27,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/fc40a2160cb54c64813929059ab26a25, entries=150, sequenceid=544, filesize=12.0 K 2024-11-14T21:34:27,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/c3974adda2f04db7ab82c1d28f6b7584 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/c3974adda2f04db7ab82c1d28f6b7584 2024-11-14T21:34:27,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/c3974adda2f04db7ab82c1d28f6b7584, entries=150, sequenceid=544, filesize=12.0 K 2024-11-14T21:34:27,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/59e9019d20cf4f76b34cc3433869377d as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/59e9019d20cf4f76b34cc3433869377d 2024-11-14T21:34:27,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/59e9019d20cf4f76b34cc3433869377d, entries=150, sequenceid=544, filesize=12.0 K 2024-11-14T21:34:27,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=46.96 KB/48090 for 81ffab6ef2cdc120372df3297622f922 in 1412ms, sequenceid=544, compaction requested=true 2024-11-14T21:34:27,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:27,760 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:27,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:34:27,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:27,760 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:27,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:34:27,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:27,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ffab6ef2cdc120372df3297622f922:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:34:27,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:27,766 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42737 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:27,766 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:27,766 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/A is initiating minor compaction (all files) 2024-11-14T21:34:27,767 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/B is initiating minor compaction (all files) 2024-11-14T21:34:27,767 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/A in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:27,767 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/B in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:27,767 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/1e055a85a0c34f1290b836b8414c8899, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/5a1d9822aafe4d998722e07e9326fbb5, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/fc40a2160cb54c64813929059ab26a25] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=41.7 K 2024-11-14T21:34:27,767 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/fe79f2bf0b544671b7433de2dd45d72b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/02fb5ac2486c4fe5aad9037586eec6ec, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/c3974adda2f04db7ab82c1d28f6b7584] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=36.9 K 2024-11-14T21:34:27,767 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting fe79f2bf0b544671b7433de2dd45d72b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731620064646 2024-11-14T21:34:27,767 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e055a85a0c34f1290b836b8414c8899, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731620064646 2024-11-14T21:34:27,767 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 02fb5ac2486c4fe5aad9037586eec6ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1731620065426 2024-11-14T21:34:27,767 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a1d9822aafe4d998722e07e9326fbb5, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1731620065388 2024-11-14T21:34:27,768 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting c3974adda2f04db7ab82c1d28f6b7584, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1731620065968 2024-11-14T21:34:27,768 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc40a2160cb54c64813929059ab26a25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1731620065968 2024-11-14T21:34:27,776 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#A#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:27,776 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/0ca67f12dbbe45fdb4e93aa0f415e383 is 50, key is test_row_0/A:col10/1731620066346/Put/seqid=0 2024-11-14T21:34:27,783 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#B#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:27,783 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/b601733a3e3d49b081804deef945c2d6 is 50, key is test_row_0/B:col10/1731620066346/Put/seqid=0 2024-11-14T21:34:27,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742364_1540 (size=13357) 2024-11-14T21:34:27,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742365_1541 (size=13323) 2024-11-14T21:34:27,799 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/b601733a3e3d49b081804deef945c2d6 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/b601733a3e3d49b081804deef945c2d6 2024-11-14T21:34:27,805 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/B of 81ffab6ef2cdc120372df3297622f922 into b601733a3e3d49b081804deef945c2d6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:27,805 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:27,805 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/B, priority=13, startTime=1731620067760; duration=0sec 2024-11-14T21:34:27,806 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:27,806 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:B 2024-11-14T21:34:27,806 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:27,807 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:27,807 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 81ffab6ef2cdc120372df3297622f922/C is initiating minor compaction (all files) 2024-11-14T21:34:27,807 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ffab6ef2cdc120372df3297622f922/C in TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:27,807 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/3036f24da4a5444eba558ef66f0e2de9, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/14ecfb30764342ffb8e47ef4e8a4c9a2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/59e9019d20cf4f76b34cc3433869377d] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp, totalSize=37.0 K 2024-11-14T21:34:27,809 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 3036f24da4a5444eba558ef66f0e2de9, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731620064646 2024-11-14T21:34:27,809 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 14ecfb30764342ffb8e47ef4e8a4c9a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1731620065426 2024-11-14T21:34:27,809 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 59e9019d20cf4f76b34cc3433869377d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1731620065968 2024-11-14T21:34:27,815 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ffab6ef2cdc120372df3297622f922#C#compaction#468 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:27,815 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/ddabaf6b5c754efda85952d5be55f324 is 50, key is test_row_0/C:col10/1731620066346/Put/seqid=0 2024-11-14T21:34:27,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742366_1542 (size=13357) 2024-11-14T21:34:27,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T21:34:27,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:27,887 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T21:34:27,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:27,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/9af4904d31ec434d93b4e276497fc2af is 50, key is test_row_0/A:col10/1731620066364/Put/seqid=0 2024-11-14T21:34:27,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742367_1543 (size=12301) 2024-11-14T21:34:27,897 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/9af4904d31ec434d93b4e276497fc2af 2024-11-14T21:34:27,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/1598484ee1fc48e0bd1106c75d7de705 is 50, key is test_row_0/B:col10/1731620066364/Put/seqid=0 2024-11-14T21:34:27,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742368_1544 (size=12301) 2024-11-14T21:34:27,915 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/1598484ee1fc48e0bd1106c75d7de705 2024-11-14T21:34:27,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/4bbb06550acc49a3b37fbc979a704dff is 50, key is test_row_0/C:col10/1731620066364/Put/seqid=0 2024-11-14T21:34:27,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742369_1545 (size=12301) 2024-11-14T21:34:28,164 INFO [Thread-2022 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2022. 2024-11-14T21:34:28,164 DEBUG [Thread-2022 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:34:28,164 DEBUG [Thread-2022 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:28,166 DEBUG [Thread-2022 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:28,166 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:34:28,166 INFO [Thread-2024 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2024. 2024-11-14T21:34:28,166 DEBUG [Thread-2024 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:34:28,166 DEBUG [Thread-2024 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:28,166 DEBUG [Thread-2024 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:28,168 INFO [Thread-2028 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2028. 2024-11-14T21:34:28,168 DEBUG [Thread-2028 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:34:28,169 DEBUG [Thread-2028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:28,169 DEBUG [Thread-2028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:28,169 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:34:28,170 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:34:28,172 INFO [Thread-2026 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2026. 2024-11-14T21:34:28,172 DEBUG [Thread-2026 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:34:28,172 DEBUG [Thread-2026 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:28,172 DEBUG [Thread-2026 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:28,172 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:34:28,172 INFO [Thread-2030 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2030. 2024-11-14T21:34:28,173 DEBUG [Thread-2030 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:34:28,173 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:28,173 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:28,173 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:34:28,196 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/0ca67f12dbbe45fdb4e93aa0f415e383 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/0ca67f12dbbe45fdb4e93aa0f415e383 2024-11-14T21:34:28,201 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/A of 81ffab6ef2cdc120372df3297622f922 into 0ca67f12dbbe45fdb4e93aa0f415e383(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:28,201 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:28,201 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/A, priority=13, startTime=1731620067760; duration=0sec 2024-11-14T21:34:28,201 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:28,201 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:A 2024-11-14T21:34:28,224 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/ddabaf6b5c754efda85952d5be55f324 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/ddabaf6b5c754efda85952d5be55f324 2024-11-14T21:34:28,228 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ffab6ef2cdc120372df3297622f922/C of 81ffab6ef2cdc120372df3297622f922 into ddabaf6b5c754efda85952d5be55f324(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:28,228 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:28,228 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., storeName=81ffab6ef2cdc120372df3297622f922/C, priority=13, startTime=1731620067760; duration=0sec 2024-11-14T21:34:28,228 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:28,228 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ffab6ef2cdc120372df3297622f922:C 2024-11-14T21:34:28,342 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/4bbb06550acc49a3b37fbc979a704dff 2024-11-14T21:34:28,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/9af4904d31ec434d93b4e276497fc2af as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/9af4904d31ec434d93b4e276497fc2af 2024-11-14T21:34:28,347 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/9af4904d31ec434d93b4e276497fc2af, entries=150, sequenceid=555, filesize=12.0 K 2024-11-14T21:34:28,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/1598484ee1fc48e0bd1106c75d7de705 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/1598484ee1fc48e0bd1106c75d7de705 2024-11-14T21:34:28,350 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/1598484ee1fc48e0bd1106c75d7de705, entries=150, sequenceid=555, filesize=12.0 K 2024-11-14T21:34:28,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/4bbb06550acc49a3b37fbc979a704dff as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/4bbb06550acc49a3b37fbc979a704dff 2024-11-14T21:34:28,353 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/4bbb06550acc49a3b37fbc979a704dff, entries=150, sequenceid=555, filesize=12.0 K 2024-11-14T21:34:28,353 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 81ffab6ef2cdc120372df3297622f922 in 466ms, sequenceid=555, compaction requested=false 2024-11-14T21:34:28,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 81ffab6ef2cdc120372df3297622f922: 2024-11-14T21:34:28,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:28,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-14T21:34:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-14T21:34:28,356 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-14T21:34:28,356 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8690 sec 2024-11-14T21:34:28,357 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.8740 sec 2024-11-14T21:34:28,592 INFO [Thread-2015 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2015. 2024-11-14T21:34:28,592 DEBUG [Thread-2015 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:34:28,592 DEBUG [Thread-2015 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:28,592 DEBUG [Thread-2015 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:28,592 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:34:28,601 INFO [Thread-2011 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2011. 2024-11-14T21:34:28,601 DEBUG [Thread-2011 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:34:28,601 DEBUG [Thread-2011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:28,602 DEBUG [Thread-2011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:28,602 INFO [Thread-2017 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2017. 2024-11-14T21:34:28,602 DEBUG [Thread-2017 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:34:28,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:34:28,602 DEBUG [Thread-2017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:28,602 DEBUG [Thread-2017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:28,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:34:28,602 INFO [Thread-2019 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2019. 2024-11-14T21:34:28,602 DEBUG [Thread-2019 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:34:28,602 DEBUG [Thread-2019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:28,602 DEBUG [Thread-2019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:28,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:34:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T21:34:28,641 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:34:37,693 INFO [Thread-2013 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2013. 2024-11-14T21:34:37,693 DEBUG [Thread-2013 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:34:37,693 DEBUG [Thread-2013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:37,693 DEBUG [Thread-2013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:37,693 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:34:37,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T21:34:37,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-14T21:34:37,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-14T21:34:37,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 107 2024-11-14T21:34:37,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-14T21:34:37,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-14T21:34:37,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T21:34:37,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T21:34:37,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 750 2024-11-14T21:34:37,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2247 rows 2024-11-14T21:34:37,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 761 2024-11-14T21:34:37,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2282 rows 2024-11-14T21:34:37,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 759 2024-11-14T21:34:37,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2261 rows 2024-11-14T21:34:37,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 765 2024-11-14T21:34:37,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2293 rows 2024-11-14T21:34:37,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 760 2024-11-14T21:34:37,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2277 rows 2024-11-14T21:34:37,694 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T21:34:37,694 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T21:34:37,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:37,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:37,694 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:34:37,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T21:34:37,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T21:34:37,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T21:34:37,699 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731620077699"}]},"ts":"1731620077699"} 2024-11-14T21:34:37,701 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T21:34:37,701 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T21:34:37,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T21:34:37,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81ffab6ef2cdc120372df3297622f922, UNASSIGN}] 2024-11-14T21:34:37,704 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81ffab6ef2cdc120372df3297622f922, UNASSIGN 2024-11-14T21:34:37,705 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=81ffab6ef2cdc120372df3297622f922, regionState=CLOSING, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:37,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81ffab6ef2cdc120372df3297622f922, UNASSIGN because future has completed 2024-11-14T21:34:37,707 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T21:34:37,707 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560}] 2024-11-14T21:34:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T21:34:37,860 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(122): Close 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:37,860 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T21:34:37,860 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1722): Closing 81ffab6ef2cdc120372df3297622f922, disabling compactions & flushes 2024-11-14T21:34:37,860 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:37,860 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:37,860 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. after waiting 0 ms 2024-11-14T21:34:37,860 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:37,860 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(2902): Flushing 81ffab6ef2cdc120372df3297622f922 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T21:34:37,860 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=A 2024-11-14T21:34:37,860 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:37,860 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=B 2024-11-14T21:34:37,860 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:37,860 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81ffab6ef2cdc120372df3297622f922, store=C 2024-11-14T21:34:37,860 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:37,872 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/3c2a3a566fe342cbbd5ae04f4337c85b is 50, key is test_row_0/A:col10/1731620077691/Put/seqid=0 2024-11-14T21:34:37,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742370_1546 (size=9857) 2024-11-14T21:34:37,880 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/3c2a3a566fe342cbbd5ae04f4337c85b 2024-11-14T21:34:37,892 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/15df920640be4a86988b2570b9869795 is 50, key is test_row_0/B:col10/1731620077691/Put/seqid=0 2024-11-14T21:34:37,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742371_1547 (size=9857) 2024-11-14T21:34:37,896 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/15df920640be4a86988b2570b9869795 2024-11-14T21:34:37,911 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/b72f505d3be04c8cb9cb746ca0fee2a0 is 50, key is test_row_0/C:col10/1731620077691/Put/seqid=0 2024-11-14T21:34:37,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742372_1548 (size=9857) 2024-11-14T21:34:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T21:34:38,317 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/b72f505d3be04c8cb9cb746ca0fee2a0 2024-11-14T21:34:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T21:34:38,324 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/A/3c2a3a566fe342cbbd5ae04f4337c85b as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/3c2a3a566fe342cbbd5ae04f4337c85b 2024-11-14T21:34:38,328 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/3c2a3a566fe342cbbd5ae04f4337c85b, entries=100, sequenceid=565, filesize=9.6 K 2024-11-14T21:34:38,334 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/B/15df920640be4a86988b2570b9869795 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/15df920640be4a86988b2570b9869795 2024-11-14T21:34:38,343 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/15df920640be4a86988b2570b9869795, entries=100, sequenceid=565, filesize=9.6 K 2024-11-14T21:34:38,344 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/.tmp/C/b72f505d3be04c8cb9cb746ca0fee2a0 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/b72f505d3be04c8cb9cb746ca0fee2a0 2024-11-14T21:34:38,363 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/b72f505d3be04c8cb9cb746ca0fee2a0, entries=100, sequenceid=565, filesize=9.6 K 2024-11-14T21:34:38,369 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 81ffab6ef2cdc120372df3297622f922 in 509ms, sequenceid=565, compaction requested=true 2024-11-14T21:34:38,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/57dd4b418f42434d90bb4af6b642f938, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/cf48cf0160f447b3a26822591859c13c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/af210329842c4032983843b9353ecb9d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/7078a538412d4fe0a0f75d7b48570092, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/4312bc54578e412f9c679c6bac0587a3, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/f627eaa85aa7459d961df4965118b6b6, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/14793bf660a147c69aae1dab4f01f49f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/4491d6dd39dc4976a3b75600bf70e951, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/a98862afab4840858cb4fdeb26a59642, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/e600a24bb50e4b3e95ce896c75c0ee39, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/65120ba626ae43a59c908676787ca62d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/9b9dc472f9a7438bb6ee2ca1ee387827, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/65d89ef0b9fb46b5af4233bdb90babdb, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/661ad75f841a408493c99b62dd7c8aac, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/268d1809fc3c4fa68a7fae07abf943f3, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/066961be56ee46389d5fc5f390918539, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/1982b06e04064c6fab6405ee0babc2a2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/840c1c8a5dcb4c9184d70a2b0d5d0ed0, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/c22b8c9e8a7547d09a9d778dc0894990, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/453ff2dfe5074590898ec1d5b1f07c7b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/5d4c20a9aa5d463cb4b9653f4b1f8bee, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/8134793ea7b540e0bdbf8eb29a5317f3, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/1f39814d231b462fae551e4ab1e7641e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/d4652d0351414d2cbb826abfaef7f0a5, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/2bb47ca28a914626ae816fa75120a93f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/9dcee99a8b9a4257b51e29a1290f74c7, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/c6b19da5b26f42a0a1fff50b532fc0f0, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/905897a2e5c542a2bb1d24ba9d8331b2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/1e055a85a0c34f1290b836b8414c8899, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/5a1d9822aafe4d998722e07e9326fbb5, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/fc40a2160cb54c64813929059ab26a25] to archive 2024-11-14T21:34:38,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:34:38,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/57dd4b418f42434d90bb4af6b642f938 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/57dd4b418f42434d90bb4af6b642f938 2024-11-14T21:34:38,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/cf48cf0160f447b3a26822591859c13c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/cf48cf0160f447b3a26822591859c13c 2024-11-14T21:34:38,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/af210329842c4032983843b9353ecb9d to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/af210329842c4032983843b9353ecb9d 2024-11-14T21:34:38,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/7078a538412d4fe0a0f75d7b48570092 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/7078a538412d4fe0a0f75d7b48570092 2024-11-14T21:34:38,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/4312bc54578e412f9c679c6bac0587a3 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/4312bc54578e412f9c679c6bac0587a3 2024-11-14T21:34:38,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/f627eaa85aa7459d961df4965118b6b6 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/f627eaa85aa7459d961df4965118b6b6 2024-11-14T21:34:38,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/14793bf660a147c69aae1dab4f01f49f to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/14793bf660a147c69aae1dab4f01f49f 2024-11-14T21:34:38,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/4491d6dd39dc4976a3b75600bf70e951 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/4491d6dd39dc4976a3b75600bf70e951 2024-11-14T21:34:38,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/a98862afab4840858cb4fdeb26a59642 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/a98862afab4840858cb4fdeb26a59642 2024-11-14T21:34:38,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/e600a24bb50e4b3e95ce896c75c0ee39 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/e600a24bb50e4b3e95ce896c75c0ee39 2024-11-14T21:34:38,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/65120ba626ae43a59c908676787ca62d to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/65120ba626ae43a59c908676787ca62d 2024-11-14T21:34:38,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/9b9dc472f9a7438bb6ee2ca1ee387827 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/9b9dc472f9a7438bb6ee2ca1ee387827 2024-11-14T21:34:38,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/65d89ef0b9fb46b5af4233bdb90babdb to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/65d89ef0b9fb46b5af4233bdb90babdb 2024-11-14T21:34:38,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/661ad75f841a408493c99b62dd7c8aac to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/661ad75f841a408493c99b62dd7c8aac 2024-11-14T21:34:38,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/268d1809fc3c4fa68a7fae07abf943f3 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/268d1809fc3c4fa68a7fae07abf943f3 2024-11-14T21:34:38,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/066961be56ee46389d5fc5f390918539 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/066961be56ee46389d5fc5f390918539 2024-11-14T21:34:38,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/1982b06e04064c6fab6405ee0babc2a2 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/1982b06e04064c6fab6405ee0babc2a2 2024-11-14T21:34:38,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/840c1c8a5dcb4c9184d70a2b0d5d0ed0 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/840c1c8a5dcb4c9184d70a2b0d5d0ed0 2024-11-14T21:34:38,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/c22b8c9e8a7547d09a9d778dc0894990 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/c22b8c9e8a7547d09a9d778dc0894990 2024-11-14T21:34:38,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/453ff2dfe5074590898ec1d5b1f07c7b to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/453ff2dfe5074590898ec1d5b1f07c7b 2024-11-14T21:34:38,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/5d4c20a9aa5d463cb4b9653f4b1f8bee to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/5d4c20a9aa5d463cb4b9653f4b1f8bee 2024-11-14T21:34:38,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/8134793ea7b540e0bdbf8eb29a5317f3 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/8134793ea7b540e0bdbf8eb29a5317f3 2024-11-14T21:34:38,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/1f39814d231b462fae551e4ab1e7641e to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/1f39814d231b462fae551e4ab1e7641e 2024-11-14T21:34:38,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/d4652d0351414d2cbb826abfaef7f0a5 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/d4652d0351414d2cbb826abfaef7f0a5 2024-11-14T21:34:38,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/2bb47ca28a914626ae816fa75120a93f to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/2bb47ca28a914626ae816fa75120a93f 2024-11-14T21:34:38,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/9dcee99a8b9a4257b51e29a1290f74c7 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/9dcee99a8b9a4257b51e29a1290f74c7 2024-11-14T21:34:38,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/c6b19da5b26f42a0a1fff50b532fc0f0 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/c6b19da5b26f42a0a1fff50b532fc0f0 2024-11-14T21:34:38,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/905897a2e5c542a2bb1d24ba9d8331b2 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/905897a2e5c542a2bb1d24ba9d8331b2 2024-11-14T21:34:38,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/1e055a85a0c34f1290b836b8414c8899 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/1e055a85a0c34f1290b836b8414c8899 2024-11-14T21:34:38,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/5a1d9822aafe4d998722e07e9326fbb5 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/5a1d9822aafe4d998722e07e9326fbb5 2024-11-14T21:34:38,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/fc40a2160cb54c64813929059ab26a25 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/fc40a2160cb54c64813929059ab26a25 2024-11-14T21:34:38,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/17f68d20f8214dee9ae8a9cc6ed0a5f6, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/30ce30323cd44fa3a093982ee44bb095, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/ac4a2c44fea44765b2af673dac315f6d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/4133e34e9c994a50801fdd2097d6dfe3, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/cc9b54698e084b19bb47dd2b79948fcd, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/b23b83ce7c44459e9824323d712c0e3b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/06f19271b69c4c3984d30a89ca2208ad, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/8c37d070cd80475b892a299d85b00f86, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/df8c19d2d8644e379e48e317c746628e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/c5a92250670e4f6394bcde64de4234d1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/a50b594c28ed4635b8d33fd3aac41ccc, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/54df9099bfd0459bb35800ad3c436f92, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/ed100804e0954c4fa6bf2d5b994dec73, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/43cefb633d614dfd8b53d8e93a4d71ab, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/5f69c66ed6cc45abafa85332985e5fce, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/097fbac7b96943738c819d5c90ce7d12, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/a0c3691791594713bdbf9e055832a150, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/e583a1d49ca7408c93dd22048ddbcd99, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/7f36536e92b24b999f39d01847276f0e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/3da56c812438491ab4ba4a514ee9616d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/a31344306fc34abfb39420e210012467, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/e41a4615be6347d0a5413276a19df10c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/b6b8807cd26149f0a446af2465f2ae6a, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/833e87a0eef04eccb02215628b8616d8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/ada4d91c69ab4800abbb84293e7f6b6c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/5682d168eda0446c82b83bd33eba7275, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/fe79f2bf0b544671b7433de2dd45d72b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/2b54376b01034f718ea69a9b3bb134ff, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/02fb5ac2486c4fe5aad9037586eec6ec, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/c3974adda2f04db7ab82c1d28f6b7584] to archive 2024-11-14T21:34:38,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:34:38,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/17f68d20f8214dee9ae8a9cc6ed0a5f6 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/17f68d20f8214dee9ae8a9cc6ed0a5f6 2024-11-14T21:34:38,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/30ce30323cd44fa3a093982ee44bb095 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/30ce30323cd44fa3a093982ee44bb095 2024-11-14T21:34:38,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/ac4a2c44fea44765b2af673dac315f6d to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/ac4a2c44fea44765b2af673dac315f6d 2024-11-14T21:34:38,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/4133e34e9c994a50801fdd2097d6dfe3 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/4133e34e9c994a50801fdd2097d6dfe3 2024-11-14T21:34:38,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/cc9b54698e084b19bb47dd2b79948fcd to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/cc9b54698e084b19bb47dd2b79948fcd 2024-11-14T21:34:38,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/b23b83ce7c44459e9824323d712c0e3b to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/b23b83ce7c44459e9824323d712c0e3b 2024-11-14T21:34:38,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/06f19271b69c4c3984d30a89ca2208ad to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/06f19271b69c4c3984d30a89ca2208ad 2024-11-14T21:34:38,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/8c37d070cd80475b892a299d85b00f86 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/8c37d070cd80475b892a299d85b00f86 2024-11-14T21:34:38,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/df8c19d2d8644e379e48e317c746628e to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/df8c19d2d8644e379e48e317c746628e 2024-11-14T21:34:38,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/c5a92250670e4f6394bcde64de4234d1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/c5a92250670e4f6394bcde64de4234d1 2024-11-14T21:34:38,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/a50b594c28ed4635b8d33fd3aac41ccc to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/a50b594c28ed4635b8d33fd3aac41ccc 2024-11-14T21:34:38,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/54df9099bfd0459bb35800ad3c436f92 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/54df9099bfd0459bb35800ad3c436f92 2024-11-14T21:34:38,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/ed100804e0954c4fa6bf2d5b994dec73 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/ed100804e0954c4fa6bf2d5b994dec73 2024-11-14T21:34:38,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/43cefb633d614dfd8b53d8e93a4d71ab to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/43cefb633d614dfd8b53d8e93a4d71ab 2024-11-14T21:34:38,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/5f69c66ed6cc45abafa85332985e5fce to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/5f69c66ed6cc45abafa85332985e5fce 2024-11-14T21:34:38,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/097fbac7b96943738c819d5c90ce7d12 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/097fbac7b96943738c819d5c90ce7d12 2024-11-14T21:34:38,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/a0c3691791594713bdbf9e055832a150 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/a0c3691791594713bdbf9e055832a150 2024-11-14T21:34:38,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/e583a1d49ca7408c93dd22048ddbcd99 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/e583a1d49ca7408c93dd22048ddbcd99 2024-11-14T21:34:38,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/7f36536e92b24b999f39d01847276f0e to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/7f36536e92b24b999f39d01847276f0e 2024-11-14T21:34:38,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/3da56c812438491ab4ba4a514ee9616d to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/3da56c812438491ab4ba4a514ee9616d 2024-11-14T21:34:38,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/a31344306fc34abfb39420e210012467 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/a31344306fc34abfb39420e210012467 2024-11-14T21:34:38,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/e41a4615be6347d0a5413276a19df10c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/e41a4615be6347d0a5413276a19df10c 2024-11-14T21:34:38,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/b6b8807cd26149f0a446af2465f2ae6a to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/b6b8807cd26149f0a446af2465f2ae6a 2024-11-14T21:34:38,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/833e87a0eef04eccb02215628b8616d8 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/833e87a0eef04eccb02215628b8616d8 2024-11-14T21:34:38,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/ada4d91c69ab4800abbb84293e7f6b6c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/ada4d91c69ab4800abbb84293e7f6b6c 2024-11-14T21:34:38,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/5682d168eda0446c82b83bd33eba7275 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/5682d168eda0446c82b83bd33eba7275 2024-11-14T21:34:38,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/fe79f2bf0b544671b7433de2dd45d72b to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/fe79f2bf0b544671b7433de2dd45d72b 2024-11-14T21:34:38,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/2b54376b01034f718ea69a9b3bb134ff to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/2b54376b01034f718ea69a9b3bb134ff 2024-11-14T21:34:38,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/02fb5ac2486c4fe5aad9037586eec6ec to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/02fb5ac2486c4fe5aad9037586eec6ec 2024-11-14T21:34:38,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/c3974adda2f04db7ab82c1d28f6b7584 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/c3974adda2f04db7ab82c1d28f6b7584 2024-11-14T21:34:38,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/fd7c5a8794da4daf9d4852e9c83ad136, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/db3368a49b914caa9b2096f5e33e2363, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/d7fd73c9bded4ce4a5b548226e21a93a, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/adc128f28a4d4cf79cb7671d14d58477, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/d2e98d9e1dd7489da2f4ecac8c14c9ec, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/b23ea78fea0c4873822cc12777c2f1b4, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/6ef91bb895af4c42aefe35c7cc7e5a71, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/69b3ea5b687f4ef685815e042f171aa2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/a7fb991a6ecd43d79ec85fb17056b30e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/0938635633a64653b27523d97b7140d7, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/cc091a71363d44779c64ba5c28208137, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/ff05576ebd8744238de7d321ac14522c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/59730e81ae344195855836053b7c13d7, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/543eb1d6495849c39acaa06d9cdb849f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/83b35f08b57e4377ba18b6f742f91495, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/43941c6e16b14074889b575104ba8bb8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/f6301a7c18d243308790cf649be28464, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/05efc3a2769a4c2e8eae37cd5139b932, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/8ef9e4f444c747cdb1cd8c0611655b4e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/15072343f6954d9a9a5ef63826a6da6e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/30fd86931f5149f593fd06e5000d61b1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/c7103aca964b4610aed2631d6d7b5842, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/fc880f45bbe1472591584df1a2724179, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/0567e5ce9dcd413dbb9b2544830d7f4e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/1c2c63a7bb814884986fade17eb67dca, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/e98cf794dcf64f60a72113dd3b91b0ed, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/4cb2d335e82b4b968f5852bade34559f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/3036f24da4a5444eba558ef66f0e2de9, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/670d4880998a42e2999fef72c66a2800, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/14ecfb30764342ffb8e47ef4e8a4c9a2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/59e9019d20cf4f76b34cc3433869377d] to archive 2024-11-14T21:34:38,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:34:38,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/fd7c5a8794da4daf9d4852e9c83ad136 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/fd7c5a8794da4daf9d4852e9c83ad136 2024-11-14T21:34:38,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/db3368a49b914caa9b2096f5e33e2363 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/db3368a49b914caa9b2096f5e33e2363 2024-11-14T21:34:38,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/d7fd73c9bded4ce4a5b548226e21a93a to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/d7fd73c9bded4ce4a5b548226e21a93a 2024-11-14T21:34:38,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/adc128f28a4d4cf79cb7671d14d58477 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/adc128f28a4d4cf79cb7671d14d58477 2024-11-14T21:34:38,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/d2e98d9e1dd7489da2f4ecac8c14c9ec to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/d2e98d9e1dd7489da2f4ecac8c14c9ec 2024-11-14T21:34:38,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/b23ea78fea0c4873822cc12777c2f1b4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/b23ea78fea0c4873822cc12777c2f1b4 2024-11-14T21:34:38,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/6ef91bb895af4c42aefe35c7cc7e5a71 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/6ef91bb895af4c42aefe35c7cc7e5a71 2024-11-14T21:34:38,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/69b3ea5b687f4ef685815e042f171aa2 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/69b3ea5b687f4ef685815e042f171aa2 2024-11-14T21:34:38,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/a7fb991a6ecd43d79ec85fb17056b30e to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/a7fb991a6ecd43d79ec85fb17056b30e 2024-11-14T21:34:38,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/0938635633a64653b27523d97b7140d7 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/0938635633a64653b27523d97b7140d7 2024-11-14T21:34:38,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/cc091a71363d44779c64ba5c28208137 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/cc091a71363d44779c64ba5c28208137 2024-11-14T21:34:38,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/ff05576ebd8744238de7d321ac14522c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/ff05576ebd8744238de7d321ac14522c 2024-11-14T21:34:38,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/59730e81ae344195855836053b7c13d7 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/59730e81ae344195855836053b7c13d7 2024-11-14T21:34:38,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/543eb1d6495849c39acaa06d9cdb849f to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/543eb1d6495849c39acaa06d9cdb849f 2024-11-14T21:34:38,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/83b35f08b57e4377ba18b6f742f91495 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/83b35f08b57e4377ba18b6f742f91495 2024-11-14T21:34:38,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/43941c6e16b14074889b575104ba8bb8 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/43941c6e16b14074889b575104ba8bb8 2024-11-14T21:34:38,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/f6301a7c18d243308790cf649be28464 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/f6301a7c18d243308790cf649be28464 2024-11-14T21:34:38,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/05efc3a2769a4c2e8eae37cd5139b932 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/05efc3a2769a4c2e8eae37cd5139b932 2024-11-14T21:34:38,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/8ef9e4f444c747cdb1cd8c0611655b4e to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/8ef9e4f444c747cdb1cd8c0611655b4e 2024-11-14T21:34:38,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/15072343f6954d9a9a5ef63826a6da6e to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/15072343f6954d9a9a5ef63826a6da6e 2024-11-14T21:34:38,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/30fd86931f5149f593fd06e5000d61b1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/30fd86931f5149f593fd06e5000d61b1 2024-11-14T21:34:38,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/c7103aca964b4610aed2631d6d7b5842 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/c7103aca964b4610aed2631d6d7b5842 2024-11-14T21:34:38,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/fc880f45bbe1472591584df1a2724179 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/fc880f45bbe1472591584df1a2724179 2024-11-14T21:34:38,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/0567e5ce9dcd413dbb9b2544830d7f4e to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/0567e5ce9dcd413dbb9b2544830d7f4e 2024-11-14T21:34:38,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/1c2c63a7bb814884986fade17eb67dca to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/1c2c63a7bb814884986fade17eb67dca 2024-11-14T21:34:38,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/e98cf794dcf64f60a72113dd3b91b0ed to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/e98cf794dcf64f60a72113dd3b91b0ed 2024-11-14T21:34:38,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/4cb2d335e82b4b968f5852bade34559f to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/4cb2d335e82b4b968f5852bade34559f 2024-11-14T21:34:38,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/3036f24da4a5444eba558ef66f0e2de9 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/3036f24da4a5444eba558ef66f0e2de9 2024-11-14T21:34:38,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/670d4880998a42e2999fef72c66a2800 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/670d4880998a42e2999fef72c66a2800 2024-11-14T21:34:38,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/14ecfb30764342ffb8e47ef4e8a4c9a2 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/14ecfb30764342ffb8e47ef4e8a4c9a2 2024-11-14T21:34:38,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/59e9019d20cf4f76b34cc3433869377d to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/59e9019d20cf4f76b34cc3433869377d 2024-11-14T21:34:38,542 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/recovered.edits/568.seqid, newMaxSeqId=568, maxSeqId=1 2024-11-14T21:34:38,543 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. 2024-11-14T21:34:38,543 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1676): Region close journal for 81ffab6ef2cdc120372df3297622f922: Waiting for close lock at 1731620077860Running coprocessor pre-close hooks at 1731620077860Disabling compacts and flushes for region at 1731620077860Disabling writes for close at 1731620077860Obtaining lock to block concurrent updates at 1731620077860Preparing flush snapshotting stores in 81ffab6ef2cdc120372df3297622f922 at 1731620077860Finished memstore snapshotting TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731620077860Flushing stores of TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922. at 1731620077861 (+1 ms)Flushing 81ffab6ef2cdc120372df3297622f922/A: creating writer at 1731620077861Flushing 81ffab6ef2cdc120372df3297622f922/A: appending metadata at 1731620077871 (+10 ms)Flushing 81ffab6ef2cdc120372df3297622f922/A: closing flushed file at 1731620077871Flushing 81ffab6ef2cdc120372df3297622f922/B: creating writer at 1731620077888 (+17 ms)Flushing 81ffab6ef2cdc120372df3297622f922/B: appending metadata at 1731620077891 (+3 ms)Flushing 81ffab6ef2cdc120372df3297622f922/B: closing flushed file at 1731620077891Flushing 81ffab6ef2cdc120372df3297622f922/C: creating writer at 1731620077901 (+10 ms)Flushing 81ffab6ef2cdc120372df3297622f922/C: appending metadata at 1731620077911 (+10 ms)Flushing 81ffab6ef2cdc120372df3297622f922/C: closing flushed file at 1731620077911Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d5bae9b: reopening flushed file at 1731620078322 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e21563: reopening flushed file at 1731620078329 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@577053fb: reopening flushed file at 1731620078343 (+14 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 81ffab6ef2cdc120372df3297622f922 in 509ms, sequenceid=565, compaction requested=true at 1731620078369 (+26 ms)Writing region close event to WAL at 1731620078539 (+170 ms)Running coprocessor post-close hooks at 1731620078543 (+4 ms)Closed at 1731620078543 2024-11-14T21:34:38,547 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(157): Closed 81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:38,548 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=81ffab6ef2cdc120372df3297622f922, regionState=CLOSED 2024-11-14T21:34:38,549 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 because future has completed 2024-11-14T21:34:38,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-14T21:34:38,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseRegionProcedure 81ffab6ef2cdc120372df3297622f922, server=ef788a4cade8,37683,1731619918560 in 843 msec 2024-11-14T21:34:38,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-14T21:34:38,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81ffab6ef2cdc120372df3297622f922, UNASSIGN in 848 msec 2024-11-14T21:34:38,555 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-14T21:34:38,555 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 852 msec 2024-11-14T21:34:38,556 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731620078556"}]},"ts":"1731620078556"} 2024-11-14T21:34:38,558 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T21:34:38,558 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T21:34:38,560 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 864 msec 2024-11-14T21:34:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T21:34:38,831 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:34:38,831 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T21:34:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:34:38,833 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:34:38,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T21:34:38,834 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=123, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:34:38,837 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:38,839 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A, FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B, FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C, FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/recovered.edits] 2024-11-14T21:34:38,841 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/0ca67f12dbbe45fdb4e93aa0f415e383 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/0ca67f12dbbe45fdb4e93aa0f415e383 2024-11-14T21:34:38,842 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/3c2a3a566fe342cbbd5ae04f4337c85b to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/3c2a3a566fe342cbbd5ae04f4337c85b 2024-11-14T21:34:38,843 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/9af4904d31ec434d93b4e276497fc2af to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/A/9af4904d31ec434d93b4e276497fc2af 2024-11-14T21:34:38,848 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/1598484ee1fc48e0bd1106c75d7de705 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/1598484ee1fc48e0bd1106c75d7de705 2024-11-14T21:34:38,850 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/15df920640be4a86988b2570b9869795 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/15df920640be4a86988b2570b9869795 2024-11-14T21:34:38,851 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/b601733a3e3d49b081804deef945c2d6 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/B/b601733a3e3d49b081804deef945c2d6 2024-11-14T21:34:38,853 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/4bbb06550acc49a3b37fbc979a704dff to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/4bbb06550acc49a3b37fbc979a704dff 2024-11-14T21:34:38,855 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/b72f505d3be04c8cb9cb746ca0fee2a0 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/b72f505d3be04c8cb9cb746ca0fee2a0 2024-11-14T21:34:38,856 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/ddabaf6b5c754efda85952d5be55f324 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/C/ddabaf6b5c754efda85952d5be55f324 2024-11-14T21:34:38,858 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/recovered.edits/568.seqid to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922/recovered.edits/568.seqid 2024-11-14T21:34:38,859 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/81ffab6ef2cdc120372df3297622f922 2024-11-14T21:34:38,859 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T21:34:38,861 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=123, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:34:38,863 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T21:34:38,866 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T21:34:38,867 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=123, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:34:38,867 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T21:34:38,867 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731620078867"}]},"ts":"9223372036854775807"} 2024-11-14T21:34:38,869 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T21:34:38,869 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 81ffab6ef2cdc120372df3297622f922, NAME => 'TestAcidGuarantees,,1731620047376.81ffab6ef2cdc120372df3297622f922.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T21:34:38,870 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T21:34:38,870 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731620078870"}]},"ts":"9223372036854775807"} 2024-11-14T21:34:38,871 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T21:34:38,872 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=123, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:34:38,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-11-14T21:34:38,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T21:34:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T21:34:38,944 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:34:38,952 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249 (was 252), OpenFileDescriptor=445 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1017 (was 1167), ProcessCount=11 (was 11), AvailableMemoryMB=777 (was 1402) 2024-11-14T21:34:38,962 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=1017, ProcessCount=11, AvailableMemoryMB=777 2024-11-14T21:34:38,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T21:34:38,964 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T21:34:38,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T21:34:38,966 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T21:34:38,966 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:38,966 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 124 2024-11-14T21:34:38,967 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T21:34:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-14T21:34:38,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742373_1549 (size=1061) 2024-11-14T21:34:39,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-14T21:34:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-14T21:34:39,374 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275 2024-11-14T21:34:39,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742374_1550 (size=53) 2024-11-14T21:34:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-14T21:34:39,808 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:34:39,808 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2c62143a7911e8dec390d4d4558f8aa4, disabling compactions & flushes 2024-11-14T21:34:39,808 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:39,808 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:39,808 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. after waiting 0 ms 2024-11-14T21:34:39,808 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:39,808 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:39,808 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2c62143a7911e8dec390d4d4558f8aa4: Waiting for close lock at 1731620079808Disabling compacts and flushes for region at 1731620079808Disabling writes for close at 1731620079808Writing region close event to WAL at 1731620079808Closed at 1731620079808 2024-11-14T21:34:39,810 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T21:34:39,810 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731620079810"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731620079810"}]},"ts":"1731620079810"} 2024-11-14T21:34:39,812 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T21:34:39,813 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T21:34:39,813 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731620079813"}]},"ts":"1731620079813"} 2024-11-14T21:34:39,815 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T21:34:39,815 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c62143a7911e8dec390d4d4558f8aa4, ASSIGN}] 2024-11-14T21:34:39,816 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c62143a7911e8dec390d4d4558f8aa4, ASSIGN 2024-11-14T21:34:39,817 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c62143a7911e8dec390d4d4558f8aa4, ASSIGN; state=OFFLINE, location=ef788a4cade8,37683,1731619918560; forceNewPlan=false, retain=false 2024-11-14T21:34:39,968 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=2c62143a7911e8dec390d4d4558f8aa4, regionState=OPENING, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:39,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c62143a7911e8dec390d4d4558f8aa4, ASSIGN because future has completed 2024-11-14T21:34:39,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560}] 2024-11-14T21:34:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-14T21:34:40,124 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:40,124 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7752): Opening region: {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:34:40,125 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:40,125 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:34:40,125 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7794): checking encryption for 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:40,125 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7797): checking classloading for 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:40,126 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:40,127 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:34:40,127 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c62143a7911e8dec390d4d4558f8aa4 columnFamilyName A 2024-11-14T21:34:40,127 DEBUG [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:40,128 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] regionserver.HStore(327): Store=2c62143a7911e8dec390d4d4558f8aa4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:34:40,128 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:40,129 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:34:40,129 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c62143a7911e8dec390d4d4558f8aa4 columnFamilyName B 2024-11-14T21:34:40,129 DEBUG [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:40,129 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] regionserver.HStore(327): Store=2c62143a7911e8dec390d4d4558f8aa4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:34:40,129 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:40,130 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:34:40,130 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c62143a7911e8dec390d4d4558f8aa4 columnFamilyName C 2024-11-14T21:34:40,130 DEBUG [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:40,131 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] regionserver.HStore(327): Store=2c62143a7911e8dec390d4d4558f8aa4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:34:40,131 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:40,131 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1038): replaying wal for 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:40,132 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:40,132 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:40,133 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1048): stopping wal replay for 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:40,133 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1060): Cleaning up temporary data for 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:40,134 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T21:34:40,135 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): writing seq id for 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:40,136 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T21:34:40,137 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1114): Opened 2c62143a7911e8dec390d4d4558f8aa4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63323039, jitterRate=-0.0564131885766983}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T21:34:40,137 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:40,137 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1006): Region open journal for 2c62143a7911e8dec390d4d4558f8aa4: Running coprocessor pre-open hook at 1731620080125Writing region info on filesystem at 1731620080125Initializing all the Stores at 1731620080125Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731620080125Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731620080126 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731620080126Cleaning up temporary data from old regions at 1731620080133 (+7 ms)Running coprocessor post-open hooks at 1731620080137 (+4 ms)Region opened successfully at 1731620080137 2024-11-14T21:34:40,138 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., pid=126, masterSystemTime=1731620080121 2024-11-14T21:34:40,140 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:40,140 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:40,141 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=2c62143a7911e8dec390d4d4558f8aa4, regionState=OPEN, openSeqNum=2, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:40,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 because future has completed 2024-11-14T21:34:40,145 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-14T21:34:40,145 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; OpenRegionProcedure 2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 in 174 msec 2024-11-14T21:34:40,147 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-14T21:34:40,147 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c62143a7911e8dec390d4d4558f8aa4, ASSIGN in 330 msec 2024-11-14T21:34:40,148 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T21:34:40,148 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731620080148"}]},"ts":"1731620080148"} 2024-11-14T21:34:40,149 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T21:34:40,150 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T21:34:40,152 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1860 sec 2024-11-14T21:34:41,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-14T21:34:41,111 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:34:41,112 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2478098, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:41,113 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:34:41,113 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:34:41,114 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:34:41,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:34:41,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:34:41,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54e7d8f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:41,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:34:41,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:34:41,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:41,116 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:34:41,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4011738e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:41,117 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:34:41,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:34:41,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:34:41,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:34:41,120 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T21:34:41,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is ef788a4cade8,38255,1731619917503 2024-11-14T21:34:41,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@401d81c0 2024-11-14T21:34:41,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T21:34:41,122 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T21:34:41,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T21:34:41,124 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T21:34:41,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T21:34:41,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742375_1551 (size=1097) 2024-11-14T21:34:41,557 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-14T21:34:41,557 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-14T21:34:41,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T21:34:41,562 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c62143a7911e8dec390d4d4558f8aa4, REOPEN/MOVE}] 2024-11-14T21:34:41,563 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c62143a7911e8dec390d4d4558f8aa4, REOPEN/MOVE 2024-11-14T21:34:41,563 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=2c62143a7911e8dec390d4d4558f8aa4, regionState=CLOSING, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:41,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c62143a7911e8dec390d4d4558f8aa4, REOPEN/MOVE because future has completed 2024-11-14T21:34:41,565 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T21:34:41,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560}] 2024-11-14T21:34:41,717 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(122): Close 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:41,717 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T21:34:41,717 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1722): Closing 2c62143a7911e8dec390d4d4558f8aa4, disabling compactions & flushes 2024-11-14T21:34:41,717 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:41,717 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:41,717 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. after waiting 0 ms 2024-11-14T21:34:41,717 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:41,721 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T21:34:41,721 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:41,721 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1676): Region close journal for 2c62143a7911e8dec390d4d4558f8aa4: Waiting for close lock at 1731620081717Running coprocessor pre-close hooks at 1731620081717Disabling compacts and flushes for region at 1731620081717Disabling writes for close at 1731620081717Writing region close event to WAL at 1731620081717Running coprocessor post-close hooks at 1731620081721 (+4 ms)Closed at 1731620081721 2024-11-14T21:34:41,722 WARN [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionServer(3299): Not adding moved region record: 2c62143a7911e8dec390d4d4558f8aa4 to self. 2024-11-14T21:34:41,724 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(157): Closed 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:41,724 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=2c62143a7911e8dec390d4d4558f8aa4, regionState=CLOSED 2024-11-14T21:34:41,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 because future has completed 2024-11-14T21:34:41,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-14T21:34:41,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseRegionProcedure 2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 in 161 msec 2024-11-14T21:34:41,729 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c62143a7911e8dec390d4d4558f8aa4, REOPEN/MOVE; state=CLOSED, location=ef788a4cade8,37683,1731619918560; forceNewPlan=false, retain=true 2024-11-14T21:34:41,879 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=2c62143a7911e8dec390d4d4558f8aa4, regionState=OPENING, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:41,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c62143a7911e8dec390d4d4558f8aa4, REOPEN/MOVE because future has completed 2024-11-14T21:34:41,882 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560}] 2024-11-14T21:34:42,037 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:42,037 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(7752): Opening region: {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:34:42,037 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:42,037 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:34:42,037 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(7794): checking encryption for 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:42,037 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(7797): checking classloading for 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:42,038 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:42,039 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:34:42,039 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c62143a7911e8dec390d4d4558f8aa4 columnFamilyName A 2024-11-14T21:34:42,040 DEBUG [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:42,041 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] regionserver.HStore(327): Store=2c62143a7911e8dec390d4d4558f8aa4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:34:42,041 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:42,042 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:34:42,042 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c62143a7911e8dec390d4d4558f8aa4 columnFamilyName B 2024-11-14T21:34:42,042 DEBUG [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:42,042 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] regionserver.HStore(327): Store=2c62143a7911e8dec390d4d4558f8aa4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:34:42,042 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:42,043 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T21:34:42,043 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c62143a7911e8dec390d4d4558f8aa4 columnFamilyName C 2024-11-14T21:34:42,043 DEBUG [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:42,043 INFO [StoreOpener-2c62143a7911e8dec390d4d4558f8aa4-1 {}] regionserver.HStore(327): Store=2c62143a7911e8dec390d4d4558f8aa4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:34:42,044 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:42,044 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1038): replaying wal for 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:42,044 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:42,051 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:42,052 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1048): stopping wal replay for 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:42,052 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1060): Cleaning up temporary data for 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:42,053 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T21:34:42,054 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1093): writing seq id for 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:42,055 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1114): Opened 2c62143a7911e8dec390d4d4558f8aa4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60743017, jitterRate=-0.09485851228237152}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T21:34:42,055 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:42,056 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1006): Region open journal for 2c62143a7911e8dec390d4d4558f8aa4: Running coprocessor pre-open hook at 1731620082037Writing region info on filesystem at 1731620082037Initializing all the Stores at 1731620082038 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731620082038Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731620082038Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731620082038Cleaning up temporary data from old regions at 1731620082052 (+14 ms)Running coprocessor post-open hooks at 1731620082055 (+3 ms)Region opened successfully at 1731620082056 (+1 ms) 2024-11-14T21:34:42,057 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., pid=131, masterSystemTime=1731620082034 2024-11-14T21:34:42,059 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:42,059 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:42,060 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=2c62143a7911e8dec390d4d4558f8aa4, regionState=OPEN, openSeqNum=5, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:42,062 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 because future has completed 2024-11-14T21:34:42,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=129 2024-11-14T21:34:42,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=129, state=SUCCESS, hasLock=false; OpenRegionProcedure 2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 in 181 msec 2024-11-14T21:34:42,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-14T21:34:42,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c62143a7911e8dec390d4d4558f8aa4, REOPEN/MOVE in 503 msec 2024-11-14T21:34:42,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-14T21:34:42,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 508 msec 2024-11-14T21:34:42,074 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 948 msec 2024-11-14T21:34:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T21:34:42,075 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:34:42,076 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@165080b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,076 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:34:42,076 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:34:42,079 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:34:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:34:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:34:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a46fdf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:34:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:34:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:42,083 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:34:42,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@420ef1d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@520fb1c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,085 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:34:42,085 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:34:42,086 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:34:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:34:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:34:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a70d083, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:34:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:34:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:42,087 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:34:42,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@261acd38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,090 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ddaf0ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,090 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:34:42,091 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:34:42,091 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:34:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:34:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:34:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@776e7de6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:34:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:34:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:42,093 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:34:42,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@448d2d55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a81e655, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,094 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:34:42,095 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:34:42,095 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:34:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:34:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:34:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48c30230, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:34:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:34:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:42,096 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:34:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604a4813, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48e1d938, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,104 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:34:42,104 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:34:42,105 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:34:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:34:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:34:42,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:42,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45ebea63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,106 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:34:42,107 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:34:42,108 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:34:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aee15e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c118b72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,110 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:34:42,110 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:34:42,111 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:34:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:34:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:34:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ef914c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:34:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:34:42,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:42,112 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:34:42,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44431fc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3012b929, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,114 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:34:42,114 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:34:42,115 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:34:42,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:34:42,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:34:42,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4751b564, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:34:42,116 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:34:42,116 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:42,119 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:34:42,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e05c34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,132 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d2f8b2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,132 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:34:42,132 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:34:42,133 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:34:42,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:34:42,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:34:42,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ab5c477, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:34:42,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:34:42,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:42,135 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:34:42,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@98b6934, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,136 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff2dcc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,136 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:34:42,136 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:34:42,137 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:34:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:34:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:34:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ace2f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:34:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:34:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:42,139 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:34:42,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41ce7ca6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25facb4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,141 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,38255,-1 for getting cluster id 2024-11-14T21:34:42,141 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:34:42,141 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '89c9f362-26c9-422a-b55f-f256516e5794' 2024-11-14T21:34:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:34:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "89c9f362-26c9-422a-b55f-f256516e5794" 2024-11-14T21:34:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bc10219, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,38255,-1] 2024-11-14T21:34:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:34:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:34:42,143 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:34:42,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64ebd0ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:34:42,144 DEBUG [Thread-2473 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:34:42,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:34:42,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:34:42,145 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:34:42,147 DEBUG [Thread-2458 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:34:42,147 DEBUG [Thread-2456 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:34:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:34:42,147 DEBUG [Thread-2475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:34:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:34:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:34:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:34:42,148 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:34:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:34:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:34:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:34:42,150 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:34:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:34:42,151 DEBUG [Thread-2469 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:34:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:34:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:34:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:34:42,153 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:34:42,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:34:42,155 DEBUG [Thread-2467 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:34:42,155 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:34:42,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:34:42,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:34:42,157 DEBUG [Thread-2462 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:34:42,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:34:42,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:34:42,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:42,158 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:34:42,158 DEBUG [Thread-2460 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:34:42,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:34:42,160 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:34:42,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:34:42,161 DEBUG [Thread-2464 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:34:42,163 DEBUG [Thread-2471 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:34:42,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:34:42,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:34:42,163 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:34:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:34:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:34:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,37683,1731619918560, seqNum=-1] 2024-11-14T21:34:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:34:42,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:34:42,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:34:42,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:34:42,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=132, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees 2024-11-14T21:34:42,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:34:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-14T21:34:42,170 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:34:42,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2c62143a7911e8dec390d4d4558f8aa4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T21:34:42,171 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=132, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:34:42,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:34:42,172 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=132, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:34:42,172 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:34:42,178 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5] 2024-11-14T21:34:42,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=A 2024-11-14T21:34:42,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:42,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=B 2024-11-14T21:34:42,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:42,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=C 2024-11-14T21:34:42,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:42,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:42,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620142203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:42,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620142205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:42,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:42,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620142206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620142202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:42,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:42,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:42,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620142206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:42,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:42,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f37a4cd9c3d5441abfe55568dc358c17_2c62143a7911e8dec390d4d4558f8aa4 is 50, key is test_row_0/A:col10/1731620082157/Put/seqid=0 2024-11-14T21:34:42,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742376_1552 (size=12154) 2024-11-14T21:34:42,263 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:42,270 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f37a4cd9c3d5441abfe55568dc358c17_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f37a4cd9c3d5441abfe55568dc358c17_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:42,275 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/545447f6859f4e83b3e13800a91b6ef2, store: [table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:42,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/545447f6859f4e83b3e13800a91b6ef2 is 175, key is test_row_0/A:col10/1731620082157/Put/seqid=0 2024-11-14T21:34:42,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-14T21:34:42,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742377_1553 (size=30955) 2024-11-14T21:34:42,282 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=19, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/545447f6859f4e83b3e13800a91b6ef2 2024-11-14T21:34:42,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-11-14T21:34:42,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/8aee2460f98d4b2db15151943146526e is 50, key is test_row_0/B:col10/1731620082157/Put/seqid=0 2024-11-14T21:34:42,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:42,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:42,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:42,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:42,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:42,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:42,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620142320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:42,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620142321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:42,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:42,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620142321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:42,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:42,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620142321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:42,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:42,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620142321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:42,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:42,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:42,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:42,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742378_1554 (size=12001) 2024-11-14T21:34:42,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=19 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/8aee2460f98d4b2db15151943146526e 2024-11-14T21:34:42,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/5bc6524593f54ff391e434133321505c is 50, key is test_row_0/C:col10/1731620082157/Put/seqid=0 2024-11-14T21:34:42,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742379_1555 (size=12001) 2024-11-14T21:34:42,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=19 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/5bc6524593f54ff391e434133321505c 2024-11-14T21:34:42,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-11-14T21:34:42,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:42,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:42,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:42,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:42,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:42,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/545447f6859f4e83b3e13800a91b6ef2 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/545447f6859f4e83b3e13800a91b6ef2 2024-11-14T21:34:42,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/545447f6859f4e83b3e13800a91b6ef2, entries=150, sequenceid=19, filesize=30.2 K 2024-11-14T21:34:42,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/8aee2460f98d4b2db15151943146526e as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/8aee2460f98d4b2db15151943146526e 2024-11-14T21:34:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-14T21:34:42,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/8aee2460f98d4b2db15151943146526e, entries=150, sequenceid=19, filesize=11.7 K 2024-11-14T21:34:42,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/5bc6524593f54ff391e434133321505c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/5bc6524593f54ff391e434133321505c 2024-11-14T21:34:42,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/5bc6524593f54ff391e434133321505c, entries=150, sequenceid=19, filesize=11.7 K 2024-11-14T21:34:42,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 2c62143a7911e8dec390d4d4558f8aa4 in 344ms, sequenceid=19, compaction requested=false 2024-11-14T21:34:42,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:42,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2c62143a7911e8dec390d4d4558f8aa4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T21:34:42,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=A 2024-11-14T21:34:42,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:42,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=B 2024-11-14T21:34:42,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:42,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=C 2024-11-14T21:34:42,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:42,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143667a069443d4ef6a7da51be2a3672e4_2c62143a7911e8dec390d4d4558f8aa4 is 50, key is test_row_0/A:col10/1731620082204/Put/seqid=0 2024-11-14T21:34:42,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:42,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620142556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:42,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620142556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:42,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620142550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:42,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620142557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:42,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620142558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:42,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:42,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742380_1556 (size=12154) 2024-11-14T21:34:42,587 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:42,591 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143667a069443d4ef6a7da51be2a3672e4_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143667a069443d4ef6a7da51be2a3672e4_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:42,596 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/35da1178ea5445baa4787629c8a7aac1, store: [table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:42,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/35da1178ea5445baa4787629c8a7aac1 is 175, key is test_row_0/A:col10/1731620082204/Put/seqid=0 2024-11-14T21:34:42,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-11-14T21:34:42,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:42,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:42,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:42,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:42,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:42,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742381_1557 (size=30955) 2024-11-14T21:34:42,638 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=39, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/35da1178ea5445baa4787629c8a7aac1 2024-11-14T21:34:42,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/58b389b9caa441e5b85675f839afdfdc is 50, key is test_row_0/B:col10/1731620082204/Put/seqid=0 2024-11-14T21:34:42,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742382_1558 (size=12001) 2024-11-14T21:34:42,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/58b389b9caa441e5b85675f839afdfdc 2024-11-14T21:34:42,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620142660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:42,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:42,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620142671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:42,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620142671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:42,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620142671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:42,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620142671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:42,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:42,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:42,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:42,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/f06fbadd06114f5ebb9109d04df94689 is 50, key is test_row_0/C:col10/1731620082204/Put/seqid=0 2024-11-14T21:34:42,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742383_1559 (size=12001) 2024-11-14T21:34:42,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/f06fbadd06114f5ebb9109d04df94689 2024-11-14T21:34:42,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/35da1178ea5445baa4787629c8a7aac1 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/35da1178ea5445baa4787629c8a7aac1 2024-11-14T21:34:42,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/35da1178ea5445baa4787629c8a7aac1, entries=150, sequenceid=39, filesize=30.2 K 2024-11-14T21:34:42,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/58b389b9caa441e5b85675f839afdfdc as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/58b389b9caa441e5b85675f839afdfdc 2024-11-14T21:34:42,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/58b389b9caa441e5b85675f839afdfdc, entries=150, sequenceid=39, filesize=11.7 K 2024-11-14T21:34:42,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/f06fbadd06114f5ebb9109d04df94689 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/f06fbadd06114f5ebb9109d04df94689 2024-11-14T21:34:42,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/f06fbadd06114f5ebb9109d04df94689, entries=150, sequenceid=39, filesize=11.7 K 2024-11-14T21:34:42,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 2c62143a7911e8dec390d4d4558f8aa4 in 233ms, sequenceid=39, compaction requested=false 2024-11-14T21:34:42,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:42,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-11-14T21:34:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:42,797 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2902): Flushing 2c62143a7911e8dec390d4d4558f8aa4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T21:34:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=A 2024-11-14T21:34:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=B 2024-11-14T21:34:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=C 2024-11-14T21:34:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-14T21:34:42,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114095805852a9548e59e4d9e585e80d0f6_2c62143a7911e8dec390d4d4558f8aa4 is 50, key is test_row_0/A:col10/1731620082557/Put/seqid=0 2024-11-14T21:34:42,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742384_1560 (size=12154) 2024-11-14T21:34:42,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:42,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:42,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:42,901 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114095805852a9548e59e4d9e585e80d0f6_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114095805852a9548e59e4d9e585e80d0f6_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:42,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/ae1a92eb47c4452da72d7112a7eb2e30, store: [table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:42,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/ae1a92eb47c4452da72d7112a7eb2e30 is 175, key is test_row_0/A:col10/1731620082557/Put/seqid=0 2024-11-14T21:34:42,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742385_1561 (size=30955) 2024-11-14T21:34:42,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731620082909, max timestamp=1731620082956Num uniques 859; 2024-11-14T21:34:42,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T21:34:42,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731620082909, max timestamp=1731620082956Num uniques 859; 2024-11-14T21:34:42,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T21:34:42,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731620082909, max timestamp=1731620082956Num uniques 859; 2024-11-14T21:34:42,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T21:34:43,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620143007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:43,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620143011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:43,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:43,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620143019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:43,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:43,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620143021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:43,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620143022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:43,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:43,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620143130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:43,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:43,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620143130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:43,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:43,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:43,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620143141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:43,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620143141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:43,142 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,142 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:43,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:43,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:43,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620143142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:43,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:43,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-14T21:34:43,330 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/ae1a92eb47c4452da72d7112a7eb2e30 2024-11-14T21:34:43,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:43,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620143340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:43,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:43,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620143342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:43,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/b98b82985aa74150b2a93aa08a7db1d2 is 50, key is test_row_0/B:col10/1731620082557/Put/seqid=0 2024-11-14T21:34:43,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:43,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:43,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620143350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:43,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:43,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620143350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:43,353 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,353 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,353 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:43,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:43,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620143362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:43,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:43,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742386_1562 (size=12001) 2024-11-14T21:34:43,374 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/b98b82985aa74150b2a93aa08a7db1d2 2024-11-14T21:34:43,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/9279bf9db8154d4fa3bdbd927a81baf7 is 50, key is test_row_0/C:col10/1731620082557/Put/seqid=0 2024-11-14T21:34:43,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742387_1563 (size=12001) 2024-11-14T21:34:43,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620143651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:43,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620143651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:43,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:43,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620143660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:43,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:43,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620143662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:43,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620143670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:43,800 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/9279bf9db8154d4fa3bdbd927a81baf7 2024-11-14T21:34:43,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/ae1a92eb47c4452da72d7112a7eb2e30 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ae1a92eb47c4452da72d7112a7eb2e30 2024-11-14T21:34:43,824 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ae1a92eb47c4452da72d7112a7eb2e30, entries=150, sequenceid=51, filesize=30.2 K 2024-11-14T21:34:43,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/b98b82985aa74150b2a93aa08a7db1d2 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/b98b82985aa74150b2a93aa08a7db1d2 2024-11-14T21:34:43,838 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/b98b82985aa74150b2a93aa08a7db1d2, entries=150, sequenceid=51, filesize=11.7 K 2024-11-14T21:34:43,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/9279bf9db8154d4fa3bdbd927a81baf7 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/9279bf9db8154d4fa3bdbd927a81baf7 2024-11-14T21:34:43,855 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/9279bf9db8154d4fa3bdbd927a81baf7, entries=150, sequenceid=51, filesize=11.7 K 2024-11-14T21:34:43,856 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 2c62143a7911e8dec390d4d4558f8aa4 in 1058ms, sequenceid=51, compaction requested=true 2024-11-14T21:34:43,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2603): Flush status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:43,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:43,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=133 2024-11-14T21:34:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=133 2024-11-14T21:34:43,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-14T21:34:43,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6850 sec 2024-11-14T21:34:43,861 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees in 1.6970 sec 2024-11-14T21:34:44,075 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T21:34:44,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2c62143a7911e8dec390d4d4558f8aa4 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-14T21:34:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:44,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=A 2024-11-14T21:34:44,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:44,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=B 2024-11-14T21:34:44,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:44,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=C 2024-11-14T21:34:44,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:44,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145ff25110ffbb4b638cfc61e41803676e_2c62143a7911e8dec390d4d4558f8aa4 is 50, key is test_row_0/A:col10/1731620084168/Put/seqid=0 2024-11-14T21:34:44,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742388_1564 (size=14594) 2024-11-14T21:34:44,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620144188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:44,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620144189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:44,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620144190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:44,197 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:44,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620144192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:44,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620144193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:44,212 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145ff25110ffbb4b638cfc61e41803676e_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145ff25110ffbb4b638cfc61e41803676e_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:44,213 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/1ad4548083d6477c8281289276e1a3e1, store: [table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:44,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/1ad4548083d6477c8281289276e1a3e1 is 175, key is test_row_0/A:col10/1731620084168/Put/seqid=0 2024-11-14T21:34:44,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742389_1565 (size=39549) 2024-11-14T21:34:44,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620144302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:44,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620144302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:44,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620144302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:44,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620144311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:44,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620144311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:44,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-14T21:34:44,332 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:34:44,332 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:34:44,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=134, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees 2024-11-14T21:34:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-14T21:34:44,334 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=134, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:34:44,335 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=134, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:34:44,335 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:34:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-14T21:34:44,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-14T21:34:44,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:44,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:44,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:44,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:44,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:44,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620144520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:44,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620144520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:44,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620144520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:44,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:44,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620144530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:44,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620144530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:44,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:44,631 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=87, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/1ad4548083d6477c8281289276e1a3e1 2024-11-14T21:34:44,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-14T21:34:44,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:44,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:44,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:44,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:44,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-14T21:34:44,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/31fcefad9eab412ca066e6ed908f7abb is 50, key is test_row_0/B:col10/1731620084168/Put/seqid=0 2024-11-14T21:34:44,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742390_1566 (size=12001) 2024-11-14T21:34:44,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-14T21:34:44,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:44,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:44,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:44,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:44,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:44,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:44,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:44,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620144831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:44,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:44,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620144831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:44,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:44,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:44,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620144831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:44,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:44,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:44,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:44,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620144841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:44,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:44,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:44,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620144851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:44,862 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,862 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:44,862 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:44,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-14T21:34:44,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-14T21:34:44,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:44,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:44,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:44,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:44,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:44,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:45,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/31fcefad9eab412ca066e6ed908f7abb 2024-11-14T21:34:45,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/248b114d164445e990659a8cd6e968d2 is 50, key is test_row_0/C:col10/1731620084168/Put/seqid=0 2024-11-14T21:34:45,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742391_1567 (size=12001) 2024-11-14T21:34:45,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/248b114d164445e990659a8cd6e968d2 2024-11-14T21:34:45,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/1ad4548083d6477c8281289276e1a3e1 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/1ad4548083d6477c8281289276e1a3e1 2024-11-14T21:34:45,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/1ad4548083d6477c8281289276e1a3e1, entries=200, sequenceid=87, filesize=38.6 K 2024-11-14T21:34:45,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/31fcefad9eab412ca066e6ed908f7abb as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/31fcefad9eab412ca066e6ed908f7abb 2024-11-14T21:34:45,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/31fcefad9eab412ca066e6ed908f7abb, entries=150, sequenceid=87, filesize=11.7 K 2024-11-14T21:34:45,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/248b114d164445e990659a8cd6e968d2 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/248b114d164445e990659a8cd6e968d2 2024-11-14T21:34:45,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/248b114d164445e990659a8cd6e968d2, entries=150, sequenceid=87, filesize=11.7 K 2024-11-14T21:34:45,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for 2c62143a7911e8dec390d4d4558f8aa4 in 947ms, sequenceid=87, compaction requested=true 2024-11-14T21:34:45,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:45,115 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:34:45,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:34:45,116 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:34:45,117 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 2c62143a7911e8dec390d4d4558f8aa4/A is initiating minor compaction (all files) 2024-11-14T21:34:45,117 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c62143a7911e8dec390d4d4558f8aa4/A in TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:45,117 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/545447f6859f4e83b3e13800a91b6ef2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/35da1178ea5445baa4787629c8a7aac1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ae1a92eb47c4452da72d7112a7eb2e30, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/1ad4548083d6477c8281289276e1a3e1] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp, totalSize=129.3 K 2024-11-14T21:34:45,117 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:45,117 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. files: [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/545447f6859f4e83b3e13800a91b6ef2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/35da1178ea5445baa4787629c8a7aac1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ae1a92eb47c4452da72d7112a7eb2e30, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/1ad4548083d6477c8281289276e1a3e1] 2024-11-14T21:34:45,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:45,117 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:34:45,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:34:45,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:45,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:34:45,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:45,118 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 545447f6859f4e83b3e13800a91b6ef2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1731620082157 2024-11-14T21:34:45,119 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35da1178ea5445baa4787629c8a7aac1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731620082203 2024-11-14T21:34:45,119 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae1a92eb47c4452da72d7112a7eb2e30, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731620082547 2024-11-14T21:34:45,119 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:34:45,119 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 2c62143a7911e8dec390d4d4558f8aa4/B is initiating minor compaction (all files) 2024-11-14T21:34:45,119 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c62143a7911e8dec390d4d4558f8aa4/B in TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:45,120 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/8aee2460f98d4b2db15151943146526e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/58b389b9caa441e5b85675f839afdfdc, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/b98b82985aa74150b2a93aa08a7db1d2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/31fcefad9eab412ca066e6ed908f7abb] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp, totalSize=46.9 K 2024-11-14T21:34:45,120 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ad4548083d6477c8281289276e1a3e1, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731620083016 2024-11-14T21:34:45,121 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 8aee2460f98d4b2db15151943146526e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1731620082157 2024-11-14T21:34:45,122 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 58b389b9caa441e5b85675f839afdfdc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731620082203 2024-11-14T21:34:45,122 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting b98b82985aa74150b2a93aa08a7db1d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731620082547 2024-11-14T21:34:45,122 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 31fcefad9eab412ca066e6ed908f7abb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731620083020 2024-11-14T21:34:45,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-14T21:34:45,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:45,126 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2902): Flushing 2c62143a7911e8dec390d4d4558f8aa4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T21:34:45,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=A 2024-11-14T21:34:45,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:45,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=B 2024-11-14T21:34:45,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:45,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=C 2024-11-14T21:34:45,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:45,148 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:45,151 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114b46b6140ea6744318d53b68bc7579764_2c62143a7911e8dec390d4d4558f8aa4 store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:45,153 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c62143a7911e8dec390d4d4558f8aa4#B#compaction#488 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:45,153 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/5f345f46158d4277a8c505dfaa2d445d is 50, key is test_row_0/B:col10/1731620084168/Put/seqid=0 2024-11-14T21:34:45,154 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114b46b6140ea6744318d53b68bc7579764_2c62143a7911e8dec390d4d4558f8aa4, store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:45,154 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b46b6140ea6744318d53b68bc7579764_2c62143a7911e8dec390d4d4558f8aa4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:45,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111460c06ea1314b4a93b6f1f3bbd0b71d3e_2c62143a7911e8dec390d4d4558f8aa4 is 50, key is test_row_0/A:col10/1731620084188/Put/seqid=0 2024-11-14T21:34:45,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742392_1568 (size=12139) 2024-11-14T21:34:45,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742393_1569 (size=4469) 2024-11-14T21:34:45,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742394_1570 (size=12154) 2024-11-14T21:34:45,201 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c62143a7911e8dec390d4d4558f8aa4#A#compaction#487 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:45,201 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/9784e0d35d9c4ae4b533647328997a1d is 175, key is test_row_0/A:col10/1731620084168/Put/seqid=0 2024-11-14T21:34:45,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742395_1571 (size=31093) 2024-11-14T21:34:45,223 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/9784e0d35d9c4ae4b533647328997a1d as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/9784e0d35d9c4ae4b533647328997a1d 2024-11-14T21:34:45,232 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2c62143a7911e8dec390d4d4558f8aa4/A of 2c62143a7911e8dec390d4d4558f8aa4 into 9784e0d35d9c4ae4b533647328997a1d(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:45,232 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:45,232 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., storeName=2c62143a7911e8dec390d4d4558f8aa4/A, priority=12, startTime=1731620085115; duration=0sec 2024-11-14T21:34:45,232 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:45,232 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:A 2024-11-14T21:34:45,232 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:34:45,233 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:34:45,233 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 2c62143a7911e8dec390d4d4558f8aa4/C is initiating minor compaction (all files) 2024-11-14T21:34:45,233 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c62143a7911e8dec390d4d4558f8aa4/C in TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:45,234 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/5bc6524593f54ff391e434133321505c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/f06fbadd06114f5ebb9109d04df94689, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/9279bf9db8154d4fa3bdbd927a81baf7, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/248b114d164445e990659a8cd6e968d2] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp, totalSize=46.9 K 2024-11-14T21:34:45,234 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bc6524593f54ff391e434133321505c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1731620082157 2024-11-14T21:34:45,234 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting f06fbadd06114f5ebb9109d04df94689, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731620082203 2024-11-14T21:34:45,234 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9279bf9db8154d4fa3bdbd927a81baf7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731620082547 2024-11-14T21:34:45,235 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 248b114d164445e990659a8cd6e968d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731620083020 2024-11-14T21:34:45,253 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c62143a7911e8dec390d4d4558f8aa4#C#compaction#490 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:45,253 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/0099dad617d44601a267b6da352c5ec3 is 50, key is test_row_0/C:col10/1731620084168/Put/seqid=0 2024-11-14T21:34:45,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742396_1572 (size=12139) 2024-11-14T21:34:45,310 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/0099dad617d44601a267b6da352c5ec3 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/0099dad617d44601a267b6da352c5ec3 2024-11-14T21:34:45,315 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2c62143a7911e8dec390d4d4558f8aa4/C of 2c62143a7911e8dec390d4d4558f8aa4 into 0099dad617d44601a267b6da352c5ec3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:45,315 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:45,315 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., storeName=2c62143a7911e8dec390d4d4558f8aa4/C, priority=12, startTime=1731620085118; duration=0sec 2024-11-14T21:34:45,315 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:45,315 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:C 2024-11-14T21:34:45,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:45,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:45,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731620085423, max timestamp=1731620085434Num uniques 859; 2024-11-14T21:34:45,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:34:45,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731620085423, max timestamp=1731620085434Num uniques 859; 2024-11-14T21:34:45,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:34:45,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731620085423, max timestamp=1731620085434Num uniques 859; 2024-11-14T21:34:45,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T21:34:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-14T21:34:45,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620145470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:45,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620145473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:45,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620145473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:45,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620145484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:45,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620145486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:45,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:45,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:45,593 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/5f345f46158d4277a8c505dfaa2d445d as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/5f345f46158d4277a8c505dfaa2d445d 2024-11-14T21:34:45,598 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2c62143a7911e8dec390d4d4558f8aa4/B of 2c62143a7911e8dec390d4d4558f8aa4 into 5f345f46158d4277a8c505dfaa2d445d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:45,598 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:45,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:45,598 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., storeName=2c62143a7911e8dec390d4d4558f8aa4/B, priority=12, startTime=1731620085117; duration=0sec 2024-11-14T21:34:45,598 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:45,598 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:B 2024-11-14T21:34:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620145592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:45,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:45,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620145593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:45,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:45,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:45,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:45,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620145593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:45,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:45,604 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111460c06ea1314b4a93b6f1f3bbd0b71d3e_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111460c06ea1314b4a93b6f1f3bbd0b71d3e_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:45,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/59eb90c1efca4d2e8cc347ccda32bbfb, store: [table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:45,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/59eb90c1efca4d2e8cc347ccda32bbfb is 175, key is test_row_0/A:col10/1731620084188/Put/seqid=0 2024-11-14T21:34:45,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620145601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:45,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:45,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620145610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:45,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742397_1573 (size=30955) 2024-11-14T21:34:45,636 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/59eb90c1efca4d2e8cc347ccda32bbfb 2024-11-14T21:34:45,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/84d0c09ed321425ba8196374df4277c1 is 50, key is test_row_0/B:col10/1731620084188/Put/seqid=0 2024-11-14T21:34:45,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742398_1574 (size=12001) 2024-11-14T21:34:45,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620145806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:45,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:45,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620145814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:45,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:45,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620145814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:45,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:45,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:45,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620145820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:45,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:45,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620145820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:45,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:45,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:46,090 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/84d0c09ed321425ba8196374df4277c1 2024-11-14T21:34:46,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/d07f165926144a3dbe09d83e43f3dd61 is 50, key is test_row_0/C:col10/1731620084188/Put/seqid=0 2024-11-14T21:34:46,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620146121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:46,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620146143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:46,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620146143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:46,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:46,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:46,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620146143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:46,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:46,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620146143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:46,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:46,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742399_1575 (size=12001) 2024-11-14T21:34:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-14T21:34:46,579 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/d07f165926144a3dbe09d83e43f3dd61 2024-11-14T21:34:46,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/59eb90c1efca4d2e8cc347ccda32bbfb as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/59eb90c1efca4d2e8cc347ccda32bbfb 2024-11-14T21:34:46,588 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/59eb90c1efca4d2e8cc347ccda32bbfb, entries=150, sequenceid=98, filesize=30.2 K 2024-11-14T21:34:46,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/84d0c09ed321425ba8196374df4277c1 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/84d0c09ed321425ba8196374df4277c1 2024-11-14T21:34:46,596 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/84d0c09ed321425ba8196374df4277c1, entries=150, sequenceid=98, filesize=11.7 K 2024-11-14T21:34:46,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/d07f165926144a3dbe09d83e43f3dd61 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d07f165926144a3dbe09d83e43f3dd61 2024-11-14T21:34:46,608 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d07f165926144a3dbe09d83e43f3dd61, entries=150, sequenceid=98, filesize=11.7 K 2024-11-14T21:34:46,609 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 2c62143a7911e8dec390d4d4558f8aa4 in 1483ms, sequenceid=98, compaction requested=false 2024-11-14T21:34:46,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2603): Flush status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:46,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:46,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=135 2024-11-14T21:34:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=135 2024-11-14T21:34:46,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-14T21:34:46,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2780 sec 2024-11-14T21:34:46,617 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees in 2.2840 sec 2024-11-14T21:34:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:46,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2c62143a7911e8dec390d4d4558f8aa4 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-14T21:34:46,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=A 2024-11-14T21:34:46,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:46,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=B 2024-11-14T21:34:46,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:46,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=C 2024-11-14T21:34:46,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:46,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111412793f684de1488caf64693f6cbe0e04_2c62143a7911e8dec390d4d4558f8aa4 is 50, key is test_row_0/A:col10/1731620085470/Put/seqid=0 2024-11-14T21:34:46,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:46,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620146661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:46,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:46,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620146661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:46,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:46,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:46,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:46,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620146662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:46,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:46,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620146683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:46,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:46,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620146683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:46,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:46,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:46,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742400_1576 (size=14794) 2024-11-14T21:34:46,734 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:46,745 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111412793f684de1488caf64693f6cbe0e04_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111412793f684de1488caf64693f6cbe0e04_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:46,746 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/54da3f03705f4caf82361fb01400953f, store: [table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:46,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/54da3f03705f4caf82361fb01400953f is 175, key is test_row_0/A:col10/1731620085470/Put/seqid=0 2024-11-14T21:34:46,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742401_1577 (size=39749) 2024-11-14T21:34:46,767 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/54da3f03705f4caf82361fb01400953f 2024-11-14T21:34:46,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/05ae41184903409e8b9f066e25ae0917 is 50, key is test_row_0/B:col10/1731620085470/Put/seqid=0 2024-11-14T21:34:46,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620146790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:46,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620146791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:46,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620146792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:46,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742402_1578 (size=12151) 2024-11-14T21:34:47,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:47,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620147000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:47,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:47,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:47,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620147000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:47,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:47,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:47,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620147010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:47,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:47,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/05ae41184903409e8b9f066e25ae0917 2024-11-14T21:34:47,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/c152f8a870484cfd9f00aca1ebd5042c is 50, key is test_row_0/C:col10/1731620085470/Put/seqid=0 2024-11-14T21:34:47,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620147311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:47,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742403_1579 (size=12151) 2024-11-14T21:34:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:47,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:47,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620147321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:47,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:47,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/c152f8a870484cfd9f00aca1ebd5042c 2024-11-14T21:34:47,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/54da3f03705f4caf82361fb01400953f as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/54da3f03705f4caf82361fb01400953f 2024-11-14T21:34:47,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:47,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620147330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:47,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:47,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/54da3f03705f4caf82361fb01400953f, entries=200, sequenceid=136, filesize=38.8 K 2024-11-14T21:34:47,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/05ae41184903409e8b9f066e25ae0917 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/05ae41184903409e8b9f066e25ae0917 2024-11-14T21:34:47,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/05ae41184903409e8b9f066e25ae0917, entries=150, sequenceid=136, filesize=11.9 K 2024-11-14T21:34:47,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/c152f8a870484cfd9f00aca1ebd5042c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/c152f8a870484cfd9f00aca1ebd5042c 2024-11-14T21:34:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/c152f8a870484cfd9f00aca1ebd5042c, entries=150, sequenceid=136, filesize=11.9 K 2024-11-14T21:34:47,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 2c62143a7911e8dec390d4d4558f8aa4 in 740ms, sequenceid=136, compaction requested=true 2024-11-14T21:34:47,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:47,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:34:47,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:47,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:34:47,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:34:47,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:34:47,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T21:34:47,379 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:47,379 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:47,394 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:47,394 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 2c62143a7911e8dec390d4d4558f8aa4/A is initiating minor compaction (all files) 2024-11-14T21:34:47,394 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c62143a7911e8dec390d4d4558f8aa4/A in TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:47,395 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/9784e0d35d9c4ae4b533647328997a1d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/59eb90c1efca4d2e8cc347ccda32bbfb, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/54da3f03705f4caf82361fb01400953f] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp, totalSize=99.4 K 2024-11-14T21:34:47,395 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:47,395 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. files: [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/9784e0d35d9c4ae4b533647328997a1d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/59eb90c1efca4d2e8cc347ccda32bbfb, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/54da3f03705f4caf82361fb01400953f] 2024-11-14T21:34:47,395 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:47,395 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 2c62143a7911e8dec390d4d4558f8aa4/C is initiating minor compaction (all files) 2024-11-14T21:34:47,395 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c62143a7911e8dec390d4d4558f8aa4/C in TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:47,395 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/0099dad617d44601a267b6da352c5ec3, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d07f165926144a3dbe09d83e43f3dd61, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/c152f8a870484cfd9f00aca1ebd5042c] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp, totalSize=35.4 K 2024-11-14T21:34:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,402 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 0099dad617d44601a267b6da352c5ec3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731620083020 2024-11-14T21:34:47,404 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9784e0d35d9c4ae4b533647328997a1d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731620083020 2024-11-14T21:34:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,410 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting d07f165926144a3dbe09d83e43f3dd61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731620084188 2024-11-14T21:34:47,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,411 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59eb90c1efca4d2e8cc347ccda32bbfb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731620084188 2024-11-14T21:34:47,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,414 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting c152f8a870484cfd9f00aca1ebd5042c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731620085470 2024-11-14T21:34:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,422 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54da3f03705f4caf82361fb01400953f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731620085462 2024-11-14T21:34:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,463 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:47,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,468 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c62143a7911e8dec390d4d4558f8aa4#C#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,474 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/e09380d449af400982f104cfea1580e6 is 50, key is test_row_0/C:col10/1731620085470/Put/seqid=0 2024-11-14T21:34:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,479 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114178da97b60b546679801b1c2bb34305b_2c62143a7911e8dec390d4d4558f8aa4 store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,481 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114178da97b60b546679801b1c2bb34305b_2c62143a7911e8dec390d4d4558f8aa4, store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:47,481 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114178da97b60b546679801b1c2bb34305b_2c62143a7911e8dec390d4d4558f8aa4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742404_1580 (size=12391) 2024-11-14T21:34:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742405_1581 (size=4469) 2024-11-14T21:34:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,527 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c62143a7911e8dec390d4d4558f8aa4#A#compaction#496 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:47,528 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/56228c7729684b3c830d5b6aa1626247 is 175, key is test_row_0/A:col10/1731620085470/Put/seqid=0 2024-11-14T21:34:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,535 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/e09380d449af400982f104cfea1580e6 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/e09380d449af400982f104cfea1580e6 2024-11-14T21:34:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,541 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c62143a7911e8dec390d4d4558f8aa4/C of 2c62143a7911e8dec390d4d4558f8aa4 into e09380d449af400982f104cfea1580e6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:47,541 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,541 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., storeName=2c62143a7911e8dec390d4d4558f8aa4/C, priority=13, startTime=1731620087379; duration=0sec 2024-11-14T21:34:47,541 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:47,541 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:C 2024-11-14T21:34:47,541 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,542 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:47,542 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 2c62143a7911e8dec390d4d4558f8aa4/B is initiating minor compaction (all files) 2024-11-14T21:34:47,542 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c62143a7911e8dec390d4d4558f8aa4/B in TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:47,542 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/5f345f46158d4277a8c505dfaa2d445d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/84d0c09ed321425ba8196374df4277c1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/05ae41184903409e8b9f066e25ae0917] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp, totalSize=35.4 K 2024-11-14T21:34:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,543 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f345f46158d4277a8c505dfaa2d445d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731620083020 2024-11-14T21:34:47,543 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 84d0c09ed321425ba8196374df4277c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731620084188 2024-11-14T21:34:47,543 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 05ae41184903409e8b9f066e25ae0917, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731620085470 2024-11-14T21:34:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,565 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c62143a7911e8dec390d4d4558f8aa4#B#compaction#498 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:47,566 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/1b061f4858c2419a848f695e65b91816 is 50, key is test_row_0/B:col10/1731620085470/Put/seqid=0 2024-11-14T21:34:47,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742406_1582 (size=31345) 2024-11-14T21:34:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,592 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/56228c7729684b3c830d5b6aa1626247 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/56228c7729684b3c830d5b6aa1626247 2024-11-14T21:34:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742407_1583 (size=12391) 2024-11-14T21:34:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,598 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c62143a7911e8dec390d4d4558f8aa4/A of 2c62143a7911e8dec390d4d4558f8aa4 into 56228c7729684b3c830d5b6aa1626247(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:47,599 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:47,599 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., storeName=2c62143a7911e8dec390d4d4558f8aa4/A, priority=13, startTime=1731620087379; duration=0sec 2024-11-14T21:34:47,599 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:47,599 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:A 2024-11-14T21:34:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,603 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/1b061f4858c2419a848f695e65b91816 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/1b061f4858c2419a848f695e65b91816 2024-11-14T21:34:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,609 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c62143a7911e8dec390d4d4558f8aa4/B of 2c62143a7911e8dec390d4d4558f8aa4 into 1b061f4858c2419a848f695e65b91816(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:47,609 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:47,609 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., storeName=2c62143a7911e8dec390d4d4558f8aa4/B, priority=13, startTime=1731620087379; duration=0sec 2024-11-14T21:34:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,609 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:47,609 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:B 2024-11-14T21:34:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:47,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2c62143a7911e8dec390d4d4558f8aa4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T21:34:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=A 2024-11-14T21:34:47,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:47,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=B 2024-11-14T21:34:47,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:47,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=C 2024-11-14T21:34:47,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fce9af9f0dfd460894648a515b4a8e49_2c62143a7911e8dec390d4d4558f8aa4 is 50, key is test_row_1/A:col10/1731620086659/Put/seqid=0 2024-11-14T21:34:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742408_1584 (size=14794) 2024-11-14T21:34:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:47,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620147788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:47,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731620087749, max timestamp=1731620087781Num uniques 859; 2024-11-14T21:34:47,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:34:47,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731620087749, max timestamp=1731620087781Num uniques 859; 2024-11-14T21:34:47,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T21:34:47,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731620087749, max timestamp=1731620087781Num uniques 859; 2024-11-14T21:34:47,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef788a4cade8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T21:34:47,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620147847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:47,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620147847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:47,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:47,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:47,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620147850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:47,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620147851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:47,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:47,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:47,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620147900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:47,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:47,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620147961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:47,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:47,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620147961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:47,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:47,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:47,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620147961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:47,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620147961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:47,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:47,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:47,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:48,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620148111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:48,138 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:48,149 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fce9af9f0dfd460894648a515b4a8e49_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fce9af9f0dfd460894648a515b4a8e49_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:48,150 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/78dae72e30e94fe5ad4d4cd2c9164316, store: [table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:48,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/78dae72e30e94fe5ad4d4cd2c9164316 is 175, key is test_row_1/A:col10/1731620086659/Put/seqid=0 2024-11-14T21:34:48,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620148170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:48,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620148172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:48,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:48,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620148181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:48,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620148181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:48,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742409_1585 (size=39745) 2024-11-14T21:34:48,197 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/78dae72e30e94fe5ad4d4cd2c9164316 2024-11-14T21:34:48,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/dbdb2d0a597743e4b890e5a34c104fae is 50, key is test_row_1/B:col10/1731620086659/Put/seqid=0 2024-11-14T21:34:48,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742410_1586 (size=9757) 2024-11-14T21:34:48,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/dbdb2d0a597743e4b890e5a34c104fae 2024-11-14T21:34:48,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/1dee0ab05eb84c87b866be5606917a59 is 50, key is test_row_1/C:col10/1731620086659/Put/seqid=0 2024-11-14T21:34:48,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742411_1587 (size=9757) 2024-11-14T21:34:48,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/1dee0ab05eb84c87b866be5606917a59 2024-11-14T21:34:48,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/78dae72e30e94fe5ad4d4cd2c9164316 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/78dae72e30e94fe5ad4d4cd2c9164316 2024-11-14T21:34:48,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/78dae72e30e94fe5ad4d4cd2c9164316, entries=200, sequenceid=151, filesize=38.8 K 2024-11-14T21:34:48,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/dbdb2d0a597743e4b890e5a34c104fae as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/dbdb2d0a597743e4b890e5a34c104fae 2024-11-14T21:34:48,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/dbdb2d0a597743e4b890e5a34c104fae, entries=100, sequenceid=151, filesize=9.5 K 2024-11-14T21:34:48,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/1dee0ab05eb84c87b866be5606917a59 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/1dee0ab05eb84c87b866be5606917a59 2024-11-14T21:34:48,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/1dee0ab05eb84c87b866be5606917a59, entries=100, sequenceid=151, filesize=9.5 K 2024-11-14T21:34:48,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.37 KB/123255 for 2c62143a7911e8dec390d4d4558f8aa4 in 669ms, sequenceid=151, compaction requested=false 2024-11-14T21:34:48,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:48,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2c62143a7911e8dec390d4d4558f8aa4 3/3 column families, dataSize=127.08 KB heapSize=364.15 KB 2024-11-14T21:34:48,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=A 2024-11-14T21:34:48,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:48,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=B 2024-11-14T21:34:48,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:48,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=C 2024-11-14T21:34:48,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:34:48,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b40818b523a84e1697ab00a009971b0f_2c62143a7911e8dec390d4d4558f8aa4 is 50, key is test_row_0/A:col10/1731620087849/Put/seqid=0 2024-11-14T21:34:48,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742412_1588 (size=12304) 2024-11-14T21:34:48,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:48,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620148484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:48,491 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:48,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620148487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:48,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:48,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620148490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:48,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620148491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:48,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620148491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:48,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:48,501 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b40818b523a84e1697ab00a009971b0f_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b40818b523a84e1697ab00a009971b0f_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-14T21:34:48,503 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:34:48,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:34:48,502 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/91f4e6362c774f4fb32740ad8665b33c, store: [table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:48,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/91f4e6362c774f4fb32740ad8665b33c is 175, key is test_row_0/A:col10/1731620087849/Put/seqid=0 2024-11-14T21:34:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees 2024-11-14T21:34:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T21:34:48,506 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:34:48,507 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=136, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:34:48,507 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:34:48,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742413_1589 (size=31105) 2024-11-14T21:34:48,519 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/91f4e6362c774f4fb32740ad8665b33c 2024-11-14T21:34:48,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/38f3375a7899472bbff0150d96b87074 is 50, key is test_row_0/B:col10/1731620087849/Put/seqid=0 2024-11-14T21:34:48,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742414_1590 (size=12151) 2024-11-14T21:34:48,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/38f3375a7899472bbff0150d96b87074 2024-11-14T21:34:48,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/36bd1486b9264d42accf5e0e2e369f98 is 50, key is test_row_0/C:col10/1731620087849/Put/seqid=0 2024-11-14T21:34:48,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620148603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T21:34:48,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742415_1591 (size=12151) 2024-11-14T21:34:48,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-14T21:34:48,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:48,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:48,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:48,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:48,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T21:34:48,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:48,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620148820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:48,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:48,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:48,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-14T21:34:48,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:48,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:48,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:48,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:48,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:48,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:49,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-14T21:34:49,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:49,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:49,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:49,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:49,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:49,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620149004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:49,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:49,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:49,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:49,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620149010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:49,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:49,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620149012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:49,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620149012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:49,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/36bd1486b9264d42accf5e0e2e369f98 2024-11-14T21:34:49,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/91f4e6362c774f4fb32740ad8665b33c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/91f4e6362c774f4fb32740ad8665b33c 2024-11-14T21:34:49,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/91f4e6362c774f4fb32740ad8665b33c, entries=150, sequenceid=187, filesize=30.4 K 2024-11-14T21:34:49,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/38f3375a7899472bbff0150d96b87074 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/38f3375a7899472bbff0150d96b87074 2024-11-14T21:34:49,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/38f3375a7899472bbff0150d96b87074, entries=150, sequenceid=187, filesize=11.9 K 2024-11-14T21:34:49,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/36bd1486b9264d42accf5e0e2e369f98 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/36bd1486b9264d42accf5e0e2e369f98 2024-11-14T21:34:49,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/36bd1486b9264d42accf5e0e2e369f98, entries=150, sequenceid=187, filesize=11.9 K 2024-11-14T21:34:49,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.08 KB/130125, heapSize ~364.10 KB/372840, currentSize=53.67 KB/54960 for 2c62143a7911e8dec390d4d4558f8aa4 in 633ms, sequenceid=187, compaction requested=true 2024-11-14T21:34:49,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:49,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:34:49,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:49,063 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:49,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:34:49,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:49,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:34:49,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:49,063 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:49,065 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:49,065 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 2c62143a7911e8dec390d4d4558f8aa4/A is initiating minor compaction (all files) 2024-11-14T21:34:49,065 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c62143a7911e8dec390d4d4558f8aa4/A in TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:49,065 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/56228c7729684b3c830d5b6aa1626247, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/78dae72e30e94fe5ad4d4cd2c9164316, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/91f4e6362c774f4fb32740ad8665b33c] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp, totalSize=99.8 K 2024-11-14T21:34:49,065 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:49,065 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. files: [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/56228c7729684b3c830d5b6aa1626247, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/78dae72e30e94fe5ad4d4cd2c9164316, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/91f4e6362c774f4fb32740ad8665b33c] 2024-11-14T21:34:49,065 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:49,065 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 2c62143a7911e8dec390d4d4558f8aa4/B is initiating minor compaction (all files) 2024-11-14T21:34:49,065 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c62143a7911e8dec390d4d4558f8aa4/B in TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:49,065 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/1b061f4858c2419a848f695e65b91816, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/dbdb2d0a597743e4b890e5a34c104fae, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/38f3375a7899472bbff0150d96b87074] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp, totalSize=33.5 K 2024-11-14T21:34:49,066 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56228c7729684b3c830d5b6aa1626247, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731620085470 2024-11-14T21:34:49,066 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b061f4858c2419a848f695e65b91816, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731620085470 2024-11-14T21:34:49,067 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78dae72e30e94fe5ad4d4cd2c9164316, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731620086659 2024-11-14T21:34:49,067 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting dbdb2d0a597743e4b890e5a34c104fae, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731620086659 2024-11-14T21:34:49,068 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 38f3375a7899472bbff0150d96b87074, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731620087848 2024-11-14T21:34:49,068 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91f4e6362c774f4fb32740ad8665b33c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731620087848 2024-11-14T21:34:49,075 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:49,076 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c62143a7911e8dec390d4d4558f8aa4#B#compaction#505 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:49,076 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/506e77835b954426a10ff71bdd169602 is 50, key is test_row_0/B:col10/1731620087849/Put/seqid=0 2024-11-14T21:34:49,078 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111409eadc7b85194d52b8612043d8cc455f_2c62143a7911e8dec390d4d4558f8aa4 store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:49,079 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111409eadc7b85194d52b8612043d8cc455f_2c62143a7911e8dec390d4d4558f8aa4, store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:49,080 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111409eadc7b85194d52b8612043d8cc455f_2c62143a7911e8dec390d4d4558f8aa4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:49,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742416_1592 (size=12493) 2024-11-14T21:34:49,092 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/506e77835b954426a10ff71bdd169602 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/506e77835b954426a10ff71bdd169602 2024-11-14T21:34:49,105 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c62143a7911e8dec390d4d4558f8aa4/B of 2c62143a7911e8dec390d4d4558f8aa4 into 506e77835b954426a10ff71bdd169602(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:49,105 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:49,105 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., storeName=2c62143a7911e8dec390d4d4558f8aa4/B, priority=13, startTime=1731620089063; duration=0sec 2024-11-14T21:34:49,105 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:49,105 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:B 2024-11-14T21:34:49,105 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:49,106 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:49,106 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 2c62143a7911e8dec390d4d4558f8aa4/C is initiating minor compaction (all files) 2024-11-14T21:34:49,106 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c62143a7911e8dec390d4d4558f8aa4/C in TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:49,106 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/e09380d449af400982f104cfea1580e6, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/1dee0ab05eb84c87b866be5606917a59, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/36bd1486b9264d42accf5e0e2e369f98] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp, totalSize=33.5 K 2024-11-14T21:34:49,110 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting e09380d449af400982f104cfea1580e6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731620085470 2024-11-14T21:34:49,112 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dee0ab05eb84c87b866be5606917a59, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731620086659 2024-11-14T21:34:49,112 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 36bd1486b9264d42accf5e0e2e369f98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731620087848 2024-11-14T21:34:49,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742417_1593 (size=4469) 2024-11-14T21:34:49,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T21:34:49,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:49,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2c62143a7911e8dec390d4d4558f8aa4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T21:34:49,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=A 2024-11-14T21:34:49,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:49,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=B 2024-11-14T21:34:49,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:49,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=C 2024-11-14T21:34:49,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:49,143 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c62143a7911e8dec390d4d4558f8aa4#C#compaction#507 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:49,143 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/84395575746244f598909d05c960eaea is 50, key is test_row_0/C:col10/1731620087849/Put/seqid=0 2024-11-14T21:34:49,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-14T21:34:49,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:49,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:49,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:49,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:49,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:49,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:49,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742418_1594 (size=12493) 2024-11-14T21:34:49,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b242676137b94d05b07e25af8d60afe0_2c62143a7911e8dec390d4d4558f8aa4 is 50, key is test_row_0/A:col10/1731620089136/Put/seqid=0 2024-11-14T21:34:49,198 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/84395575746244f598909d05c960eaea as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/84395575746244f598909d05c960eaea 2024-11-14T21:34:49,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742419_1595 (size=14794) 2024-11-14T21:34:49,225 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c62143a7911e8dec390d4d4558f8aa4/C of 2c62143a7911e8dec390d4d4558f8aa4 into 84395575746244f598909d05c960eaea(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:49,225 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:49,225 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., storeName=2c62143a7911e8dec390d4d4558f8aa4/C, priority=13, startTime=1731620089063; duration=0sec 2024-11-14T21:34:49,225 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:49,225 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:C 2024-11-14T21:34:49,228 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:49,243 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b242676137b94d05b07e25af8d60afe0_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b242676137b94d05b07e25af8d60afe0_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:49,246 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/c0c07f51827e4578a10d8fd09b1c674b, store: [table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:49,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/c0c07f51827e4578a10d8fd09b1c674b is 175, key is test_row_0/A:col10/1731620089136/Put/seqid=0 2024-11-14T21:34:49,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742420_1596 (size=39749) 2024-11-14T21:34:49,259 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/c0c07f51827e4578a10d8fd09b1c674b 2024-11-14T21:34:49,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/13b759ca7f674c1b9fe356b4560d5a3c is 50, key is test_row_0/B:col10/1731620089136/Put/seqid=0 2024-11-14T21:34:49,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:49,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620149290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:49,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742421_1597 (size=12151) 2024-11-14T21:34:49,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-14T21:34:49,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:49,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:49,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:49,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:49,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:49,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:49,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:49,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620149402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:49,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:49,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:49,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:49,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-14T21:34:49,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:49,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:49,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:49,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:49,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:49,532 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c62143a7911e8dec390d4d4558f8aa4#A#compaction#506 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:49,533 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/ef6e007c369a4807962c04097d6da0da is 175, key is test_row_0/A:col10/1731620087849/Put/seqid=0 2024-11-14T21:34:49,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742422_1598 (size=31447) 2024-11-14T21:34:49,573 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/ef6e007c369a4807962c04097d6da0da as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ef6e007c369a4807962c04097d6da0da 2024-11-14T21:34:49,587 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c62143a7911e8dec390d4d4558f8aa4/A of 2c62143a7911e8dec390d4d4558f8aa4 into ef6e007c369a4807962c04097d6da0da(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:49,587 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:49,587 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., storeName=2c62143a7911e8dec390d4d4558f8aa4/A, priority=13, startTime=1731620089063; duration=0sec 2024-11-14T21:34:49,587 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:49,587 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:A 2024-11-14T21:34:49,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620149620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:49,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:49,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:49,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:49,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-14T21:34:49,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:49,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:49,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:49,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:49,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T21:34:49,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/13b759ca7f674c1b9fe356b4560d5a3c 2024-11-14T21:34:49,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/d27bf48fd1ea45a989c53a26e43e6d36 is 50, key is test_row_0/C:col10/1731620089136/Put/seqid=0 2024-11-14T21:34:49,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742423_1599 (size=12151) 2024-11-14T21:34:49,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-14T21:34:49,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:49,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:49,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:49,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:49,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:49,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620149932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:49,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-14T21:34:49,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:49,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:49,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:49,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:49,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:50,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620150023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:50,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620150023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:50,029 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_2' in 2c62143a7911e8dec390d4d4558f8aa4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:34:50,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:50,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:50,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:50,030 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_0' in 2c62143a7911e8dec390d4d4558f8aa4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:34:50,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:50,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:50,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:50,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620150032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:50,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620150032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:50,037 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_2' in 2c62143a7911e8dec390d4d4558f8aa4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:34:50,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:50,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:50,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:50,038 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_2' in 2c62143a7911e8dec390d4d4558f8aa4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:34:50,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:50,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:50,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:50,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-14T21:34:50,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:50,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:50,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:50,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:50,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:50,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/d27bf48fd1ea45a989c53a26e43e6d36 2024-11-14T21:34:50,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/c0c07f51827e4578a10d8fd09b1c674b as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/c0c07f51827e4578a10d8fd09b1c674b 2024-11-14T21:34:50,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/c0c07f51827e4578a10d8fd09b1c674b, entries=200, sequenceid=200, filesize=38.8 K 2024-11-14T21:34:50,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/13b759ca7f674c1b9fe356b4560d5a3c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/13b759ca7f674c1b9fe356b4560d5a3c 2024-11-14T21:34:50,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/13b759ca7f674c1b9fe356b4560d5a3c, entries=150, sequenceid=200, filesize=11.9 K 2024-11-14T21:34:50,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/d27bf48fd1ea45a989c53a26e43e6d36 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d27bf48fd1ea45a989c53a26e43e6d36 2024-11-14T21:34:50,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d27bf48fd1ea45a989c53a26e43e6d36, entries=150, sequenceid=200, filesize=11.9 K 2024-11-14T21:34:50,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 2c62143a7911e8dec390d4d4558f8aa4 in 1017ms, sequenceid=200, compaction requested=false 2024-11-14T21:34:50,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:50,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-14T21:34:50,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:50,270 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2902): Flushing 2c62143a7911e8dec390d4d4558f8aa4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T21:34:50,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=A 2024-11-14T21:34:50,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:50,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=B 2024-11-14T21:34:50,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:50,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=C 2024-11-14T21:34:50,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:50,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bf4f9bd212744f8bba319e5898a8ca7e_2c62143a7911e8dec390d4d4558f8aa4 is 50, key is test_row_0/A:col10/1731620089213/Put/seqid=0 2024-11-14T21:34:50,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742424_1600 (size=12304) 2024-11-14T21:34:50,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:50,370 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bf4f9bd212744f8bba319e5898a8ca7e_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bf4f9bd212744f8bba319e5898a8ca7e_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:50,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/ec7da7282a344cbd8b35d505fc5e81a6, store: [table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:50,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/ec7da7282a344cbd8b35d505fc5e81a6 is 175, key is test_row_0/A:col10/1731620089213/Put/seqid=0 2024-11-14T21:34:50,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742425_1601 (size=31105) 2024-11-14T21:34:50,408 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/ec7da7282a344cbd8b35d505fc5e81a6 2024-11-14T21:34:50,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/196ec192d74040e3b5c0c055086070db is 50, key is test_row_0/B:col10/1731620089213/Put/seqid=0 2024-11-14T21:34:50,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:50,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:50,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742426_1602 (size=12151) 2024-11-14T21:34:50,462 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/196ec192d74040e3b5c0c055086070db 2024-11-14T21:34:50,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/f9846e3f9b2745b4a07fa3ab21d451d5 is 50, key is test_row_0/C:col10/1731620089213/Put/seqid=0 2024-11-14T21:34:50,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742427_1603 (size=12151) 2024-11-14T21:34:50,521 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/f9846e3f9b2745b4a07fa3ab21d451d5 2024-11-14T21:34:50,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/ec7da7282a344cbd8b35d505fc5e81a6 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ec7da7282a344cbd8b35d505fc5e81a6 2024-11-14T21:34:50,535 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ec7da7282a344cbd8b35d505fc5e81a6, entries=150, sequenceid=222, filesize=30.4 K 2024-11-14T21:34:50,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/196ec192d74040e3b5c0c055086070db as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/196ec192d74040e3b5c0c055086070db 2024-11-14T21:34:50,539 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/196ec192d74040e3b5c0c055086070db, entries=150, sequenceid=222, filesize=11.9 K 2024-11-14T21:34:50,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/f9846e3f9b2745b4a07fa3ab21d451d5 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/f9846e3f9b2745b4a07fa3ab21d451d5 2024-11-14T21:34:50,546 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/f9846e3f9b2745b4a07fa3ab21d451d5, entries=150, sequenceid=222, filesize=11.9 K 2024-11-14T21:34:50,547 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=53.67 KB/54960 for 2c62143a7911e8dec390d4d4558f8aa4 in 276ms, sequenceid=222, compaction requested=true 2024-11-14T21:34:50,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2603): Flush status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:50,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:50,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=137 2024-11-14T21:34:50,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=137 2024-11-14T21:34:50,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-14T21:34:50,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0420 sec 2024-11-14T21:34:50,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees in 2.0510 sec 2024-11-14T21:34:50,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:50,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2c62143a7911e8dec390d4d4558f8aa4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T21:34:50,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=A 2024-11-14T21:34:50,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:50,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=B 2024-11-14T21:34:50,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:50,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=C 2024-11-14T21:34:50,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:50,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149dcded3380a442efbfa136df243c4003_2c62143a7911e8dec390d4d4558f8aa4 is 50, key is test_row_0/A:col10/1731620090520/Put/seqid=0 2024-11-14T21:34:50,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742428_1604 (size=12304) 2024-11-14T21:34:50,626 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:50,631 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149dcded3380a442efbfa136df243c4003_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149dcded3380a442efbfa136df243c4003_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:50,640 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/db2a9ff078c7486aa825b22121e5cb1f, store: [table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:50,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/db2a9ff078c7486aa825b22121e5cb1f is 175, key is test_row_0/A:col10/1731620090520/Put/seqid=0 2024-11-14T21:34:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T21:34:50,650 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:34:50,651 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:34:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees 2024-11-14T21:34:50,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-14T21:34:50,653 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:34:50,654 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:34:50,654 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:34:50,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742429_1605 (size=31105) 2024-11-14T21:34:50,693 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/db2a9ff078c7486aa825b22121e5cb1f 2024-11-14T21:34:50,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/195d24beeb64410189ef67706ce261c6 is 50, key is test_row_0/B:col10/1731620090520/Put/seqid=0 2024-11-14T21:34:50,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620150712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:50,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:50,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:50,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:50,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742430_1606 (size=12151) 2024-11-14T21:34:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-14T21:34:50,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/195d24beeb64410189ef67706ce261c6 2024-11-14T21:34:50,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/e636d7be966a401381e2c55f5cbb65e9 is 50, key is test_row_0/C:col10/1731620090520/Put/seqid=0 2024-11-14T21:34:50,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-14T21:34:50,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:50,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:50,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:50,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:50,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:50,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742431_1607 (size=12151) 2024-11-14T21:34:50,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620150831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:50,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-14T21:34:50,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:50,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:50,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:50,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:50,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-14T21:34:51,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620151040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:51,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-14T21:34:51,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:51,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:51,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:51,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:51,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:51,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:51,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/e636d7be966a401381e2c55f5cbb65e9 2024-11-14T21:34:51,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/db2a9ff078c7486aa825b22121e5cb1f as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/db2a9ff078c7486aa825b22121e5cb1f 2024-11-14T21:34:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-14T21:34:51,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-14T21:34:51,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:51,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:51,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:51,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:51,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:51,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/db2a9ff078c7486aa825b22121e5cb1f, entries=150, sequenceid=234, filesize=30.4 K 2024-11-14T21:34:51,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/195d24beeb64410189ef67706ce261c6 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/195d24beeb64410189ef67706ce261c6 2024-11-14T21:34:51,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/195d24beeb64410189ef67706ce261c6, entries=150, sequenceid=234, filesize=11.9 K 2024-11-14T21:34:51,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/e636d7be966a401381e2c55f5cbb65e9 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/e636d7be966a401381e2c55f5cbb65e9 2024-11-14T21:34:51,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620151351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:51,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:51,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:51,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:51,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/e636d7be966a401381e2c55f5cbb65e9, entries=150, sequenceid=234, filesize=11.9 K 2024-11-14T21:34:51,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 2c62143a7911e8dec390d4d4558f8aa4 in 795ms, sequenceid=234, compaction requested=true 2024-11-14T21:34:51,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:51,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:34:51,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:51,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:34:51,359 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:34:51,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:51,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:34:51,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:34:51,359 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:34:51,371 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:34:51,371 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 2c62143a7911e8dec390d4d4558f8aa4/B is initiating minor compaction (all files) 2024-11-14T21:34:51,371 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c62143a7911e8dec390d4d4558f8aa4/B in TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:51,371 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/506e77835b954426a10ff71bdd169602, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/13b759ca7f674c1b9fe356b4560d5a3c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/196ec192d74040e3b5c0c055086070db, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/195d24beeb64410189ef67706ce261c6] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp, totalSize=47.8 K 2024-11-14T21:34:51,372 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 506e77835b954426a10ff71bdd169602, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731620087848 2024-11-14T21:34:51,372 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 13b759ca7f674c1b9fe356b4560d5a3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731620088463 2024-11-14T21:34:51,372 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133406 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:34:51,372 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 2c62143a7911e8dec390d4d4558f8aa4/A is initiating minor compaction (all files) 2024-11-14T21:34:51,372 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c62143a7911e8dec390d4d4558f8aa4/A in TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:51,373 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ef6e007c369a4807962c04097d6da0da, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/c0c07f51827e4578a10d8fd09b1c674b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ec7da7282a344cbd8b35d505fc5e81a6, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/db2a9ff078c7486aa825b22121e5cb1f] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp, totalSize=130.3 K 2024-11-14T21:34:51,373 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:51,373 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. files: [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ef6e007c369a4807962c04097d6da0da, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/c0c07f51827e4578a10d8fd09b1c674b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ec7da7282a344cbd8b35d505fc5e81a6, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/db2a9ff078c7486aa825b22121e5cb1f] 2024-11-14T21:34:51,377 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef6e007c369a4807962c04097d6da0da, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731620087848 2024-11-14T21:34:51,378 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 196ec192d74040e3b5c0c055086070db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731620089213 2024-11-14T21:34:51,378 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0c07f51827e4578a10d8fd09b1c674b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731620088456 2024-11-14T21:34:51,378 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 195d24beeb64410189ef67706ce261c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731620090520 2024-11-14T21:34:51,379 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec7da7282a344cbd8b35d505fc5e81a6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731620089213 2024-11-14T21:34:51,380 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting db2a9ff078c7486aa825b22121e5cb1f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731620090520 2024-11-14T21:34:51,405 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c62143a7911e8dec390d4d4558f8aa4#B#compaction#517 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:51,406 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/70390e339293428c9f9b37fabf62219b is 50, key is test_row_0/B:col10/1731620090520/Put/seqid=0 2024-11-14T21:34:51,415 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:51,435 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411149d44b87066c54e27ba38e65339dcd446_2c62143a7911e8dec390d4d4558f8aa4 store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:51,437 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411149d44b87066c54e27ba38e65339dcd446_2c62143a7911e8dec390d4d4558f8aa4, store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:51,438 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149d44b87066c54e27ba38e65339dcd446_2c62143a7911e8dec390d4d4558f8aa4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:51,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-14T21:34:51,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:51,455 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2902): Flushing 2c62143a7911e8dec390d4d4558f8aa4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T21:34:51,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742432_1608 (size=12629) 2024-11-14T21:34:51,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=A 2024-11-14T21:34:51,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:51,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=B 2024-11-14T21:34:51,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:51,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=C 2024-11-14T21:34:51,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:51,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fcbbc158b46d451ca7dc5c955700a3d9_2c62143a7911e8dec390d4d4558f8aa4 is 50, key is test_row_0/A:col10/1731620090700/Put/seqid=0 2024-11-14T21:34:51,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742433_1609 (size=4469) 2024-11-14T21:34:51,477 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/70390e339293428c9f9b37fabf62219b as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/70390e339293428c9f9b37fabf62219b 2024-11-14T21:34:51,477 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c62143a7911e8dec390d4d4558f8aa4#A#compaction#518 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:51,478 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/b25db12053864b4b9346206c8db57594 is 175, key is test_row_0/A:col10/1731620090520/Put/seqid=0 2024-11-14T21:34:51,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742434_1610 (size=12304) 2024-11-14T21:34:51,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742435_1611 (size=31583) 2024-11-14T21:34:51,498 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2c62143a7911e8dec390d4d4558f8aa4/B of 2c62143a7911e8dec390d4d4558f8aa4 into 70390e339293428c9f9b37fabf62219b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:51,498 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:51,498 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., storeName=2c62143a7911e8dec390d4d4558f8aa4/B, priority=12, startTime=1731620091359; duration=0sec 2024-11-14T21:34:51,498 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:51,498 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:B 2024-11-14T21:34:51,498 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:34:51,499 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T21:34:51,499 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 2c62143a7911e8dec390d4d4558f8aa4/C is initiating minor compaction (all files) 2024-11-14T21:34:51,499 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c62143a7911e8dec390d4d4558f8aa4/C in TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:51,500 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/84395575746244f598909d05c960eaea, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d27bf48fd1ea45a989c53a26e43e6d36, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/f9846e3f9b2745b4a07fa3ab21d451d5, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/e636d7be966a401381e2c55f5cbb65e9] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp, totalSize=47.8 K 2024-11-14T21:34:51,500 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 84395575746244f598909d05c960eaea, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731620087848 2024-11-14T21:34:51,500 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting d27bf48fd1ea45a989c53a26e43e6d36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731620088463 2024-11-14T21:34:51,500 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting f9846e3f9b2745b4a07fa3ab21d451d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731620089213 2024-11-14T21:34:51,501 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting e636d7be966a401381e2c55f5cbb65e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731620090520 2024-11-14T21:34:51,522 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c62143a7911e8dec390d4d4558f8aa4#C#compaction#520 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:51,523 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/244191d1444a47208f33ad9d2477d481 is 50, key is test_row_0/C:col10/1731620090520/Put/seqid=0 2024-11-14T21:34:51,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742436_1612 (size=12629) 2024-11-14T21:34:51,538 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/244191d1444a47208f33ad9d2477d481 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/244191d1444a47208f33ad9d2477d481 2024-11-14T21:34:51,550 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2c62143a7911e8dec390d4d4558f8aa4/C of 2c62143a7911e8dec390d4d4558f8aa4 into 244191d1444a47208f33ad9d2477d481(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:51,550 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:51,550 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., storeName=2c62143a7911e8dec390d4d4558f8aa4/C, priority=12, startTime=1731620091359; duration=0sec 2024-11-14T21:34:51,550 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:51,550 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:C 2024-11-14T21:34:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-14T21:34:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:51,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:51,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:51,903 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fcbbc158b46d451ca7dc5c955700a3d9_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fcbbc158b46d451ca7dc5c955700a3d9_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:51,909 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/b25db12053864b4b9346206c8db57594 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/b25db12053864b4b9346206c8db57594 2024-11-14T21:34:51,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/4f621884b8bb4b82a6ef440fffed828e, store: [table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:51,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/4f621884b8bb4b82a6ef440fffed828e is 175, key is test_row_0/A:col10/1731620090700/Put/seqid=0 2024-11-14T21:34:51,916 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2c62143a7911e8dec390d4d4558f8aa4/A of 2c62143a7911e8dec390d4d4558f8aa4 into b25db12053864b4b9346206c8db57594(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:51,916 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:51,916 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., storeName=2c62143a7911e8dec390d4d4558f8aa4/A, priority=12, startTime=1731620091359; duration=0sec 2024-11-14T21:34:51,916 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:51,916 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:A 2024-11-14T21:34:51,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742437_1613 (size=31105) 2024-11-14T21:34:51,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:51,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620151937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:51,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:51,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:51,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:51,945 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/4f621884b8bb4b82a6ef440fffed828e 2024-11-14T21:34:51,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/1fc7d0dc4c8046a29e90709f4bda6dc9 is 50, key is test_row_0/B:col10/1731620090700/Put/seqid=0 2024-11-14T21:34:51,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742438_1614 (size=12151) 2024-11-14T21:34:52,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:52,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620152042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:52,050 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_0' in 2c62143a7911e8dec390d4d4558f8aa4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:34:52,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:52,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:52,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:52,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620152052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:52,054 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_2' in 2c62143a7911e8dec390d4d4558f8aa4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:34:52,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:52,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:52,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620152052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:52,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:52,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:52,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620152060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:52,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:52,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620152062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:52,064 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_2' in 2c62143a7911e8dec390d4d4558f8aa4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:34:52,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:52,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:52,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:52,066 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_2' in 2c62143a7911e8dec390d4d4558f8aa4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:34:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:52,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620152261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:52,371 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/1fc7d0dc4c8046a29e90709f4bda6dc9 2024-11-14T21:34:52,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/3e6df6fc519f4342aca75a6c0a0f59f2 is 50, key is test_row_0/C:col10/1731620090700/Put/seqid=0 2024-11-14T21:34:52,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742439_1615 (size=12151) 2024-11-14T21:34:52,420 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/3e6df6fc519f4342aca75a6c0a0f59f2 2024-11-14T21:34:52,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/4f621884b8bb4b82a6ef440fffed828e as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/4f621884b8bb4b82a6ef440fffed828e 2024-11-14T21:34:52,467 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/4f621884b8bb4b82a6ef440fffed828e, entries=150, sequenceid=254, filesize=30.4 K 2024-11-14T21:34:52,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/1fc7d0dc4c8046a29e90709f4bda6dc9 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/1fc7d0dc4c8046a29e90709f4bda6dc9 2024-11-14T21:34:52,486 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/1fc7d0dc4c8046a29e90709f4bda6dc9, entries=150, sequenceid=254, filesize=11.9 K 2024-11-14T21:34:52,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/3e6df6fc519f4342aca75a6c0a0f59f2 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/3e6df6fc519f4342aca75a6c0a0f59f2 2024-11-14T21:34:52,495 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/3e6df6fc519f4342aca75a6c0a0f59f2, entries=150, sequenceid=254, filesize=11.9 K 2024-11-14T21:34:52,503 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 2c62143a7911e8dec390d4d4558f8aa4 in 1048ms, sequenceid=254, compaction requested=false 2024-11-14T21:34:52,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2603): Flush status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:52,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:52,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=139 2024-11-14T21:34:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=139 2024-11-14T21:34:52,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-14T21:34:52,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8650 sec 2024-11-14T21:34:52,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees in 1.8890 sec 2024-11-14T21:34:52,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2c62143a7911e8dec390d4d4558f8aa4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T21:34:52,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=A 2024-11-14T21:34:52,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:52,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=B 2024-11-14T21:34:52,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:52,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=C 2024-11-14T21:34:52,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:52,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f972713ada1c4094923073bad024abd9_2c62143a7911e8dec390d4d4558f8aa4 is 50, key is test_row_0/A:col10/1731620091932/Put/seqid=0 2024-11-14T21:34:52,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742440_1616 (size=14994) 2024-11-14T21:34:52,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620152727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:52,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:52,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:52,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-14T21:34:52,831 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:34:52,831 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:34:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-11-14T21:34:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T21:34:52,833 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:34:52,834 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:34:52,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:34:52,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:52,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620152847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:52,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:52,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:52,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T21:34:52,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-14T21:34:52,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:52,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:52,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:52,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:52,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:53,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:53,059 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:53,063 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f972713ada1c4094923073bad024abd9_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f972713ada1c4094923073bad024abd9_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:53,064 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/ac80965ac9094dd1a5d2a7b7815404e1, store: [table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:53,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/ac80965ac9094dd1a5d2a7b7815404e1 is 175, key is test_row_0/A:col10/1731620091932/Put/seqid=0 2024-11-14T21:34:53,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:53,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620153061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:53,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:53,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:53,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:53,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742441_1617 (size=39949) 2024-11-14T21:34:53,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T21:34:53,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-14T21:34:53,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:53,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:53,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:53,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:53,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:53,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-14T21:34:53,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:53,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:53,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:53,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:53,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:53,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:53,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:53,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620153371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:53,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:53,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:53,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:53,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T21:34:53,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-14T21:34:53,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:53,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:53,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:53,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:53,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:53,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:53,473 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/ac80965ac9094dd1a5d2a7b7815404e1 2024-11-14T21:34:53,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/8a9d12804525460eb69a2fbd5bbf512a is 50, key is test_row_0/B:col10/1731620091932/Put/seqid=0 2024-11-14T21:34:53,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742442_1618 (size=12301) 2024-11-14T21:34:53,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-14T21:34:53,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:53,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:53,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:53,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:53,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:53,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:53,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-14T21:34:53,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:53,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:53,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:53,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:53,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:53,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620153880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:53,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/8a9d12804525460eb69a2fbd5bbf512a 2024-11-14T21:34:53,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/d50702ff4470463dac4498caea46f88a is 50, key is test_row_0/C:col10/1731620091932/Put/seqid=0 2024-11-14T21:34:53,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742443_1619 (size=12301) 2024-11-14T21:34:53,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-14T21:34:53,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:53,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:53,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:53,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:53,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T21:34:54,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-14T21:34:54,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:54,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:54,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:54,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:54,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:54,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-14T21:34:54,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:54,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:54,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:54,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:54,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:54,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/d50702ff4470463dac4498caea46f88a 2024-11-14T21:34:54,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/ac80965ac9094dd1a5d2a7b7815404e1 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ac80965ac9094dd1a5d2a7b7815404e1 2024-11-14T21:34:54,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ac80965ac9094dd1a5d2a7b7815404e1, entries=200, sequenceid=270, filesize=39.0 K 2024-11-14T21:34:54,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/8a9d12804525460eb69a2fbd5bbf512a as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/8a9d12804525460eb69a2fbd5bbf512a 2024-11-14T21:34:54,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/8a9d12804525460eb69a2fbd5bbf512a, entries=150, sequenceid=270, filesize=12.0 K 2024-11-14T21:34:54,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/d50702ff4470463dac4498caea46f88a as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d50702ff4470463dac4498caea46f88a 2024-11-14T21:34:54,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d50702ff4470463dac4498caea46f88a, entries=150, sequenceid=270, filesize=12.0 K 2024-11-14T21:34:54,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 2c62143a7911e8dec390d4d4558f8aa4 in 1808ms, sequenceid=270, compaction requested=true 2024-11-14T21:34:54,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:54,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:34:54,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:54,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:34:54,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:54,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:34:54,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:34:54,393 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:54,393 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102637 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:54,394 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 2c62143a7911e8dec390d4d4558f8aa4/A is initiating minor compaction (all files) 2024-11-14T21:34:54,394 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c62143a7911e8dec390d4d4558f8aa4/A in TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:54,394 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/b25db12053864b4b9346206c8db57594, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/4f621884b8bb4b82a6ef440fffed828e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ac80965ac9094dd1a5d2a7b7815404e1] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp, totalSize=100.2 K 2024-11-14T21:34:54,394 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:54,394 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. files: [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/b25db12053864b4b9346206c8db57594, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/4f621884b8bb4b82a6ef440fffed828e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ac80965ac9094dd1a5d2a7b7815404e1] 2024-11-14T21:34:54,394 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:54,395 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting b25db12053864b4b9346206c8db57594, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731620090520 2024-11-14T21:34:54,396 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f621884b8bb4b82a6ef440fffed828e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731620090700 2024-11-14T21:34:54,397 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting ac80965ac9094dd1a5d2a7b7815404e1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731620091903 2024-11-14T21:34:54,410 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:54,410 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 2c62143a7911e8dec390d4d4558f8aa4/B is initiating minor compaction (all files) 2024-11-14T21:34:54,411 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c62143a7911e8dec390d4d4558f8aa4/B in TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:54,411 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/70390e339293428c9f9b37fabf62219b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/1fc7d0dc4c8046a29e90709f4bda6dc9, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/8a9d12804525460eb69a2fbd5bbf512a] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp, totalSize=36.2 K 2024-11-14T21:34:54,414 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70390e339293428c9f9b37fabf62219b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731620090520 2024-11-14T21:34:54,418 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fc7d0dc4c8046a29e90709f4bda6dc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731620090700 2024-11-14T21:34:54,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-14T21:34:54,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:54,420 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing 2c62143a7911e8dec390d4d4558f8aa4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T21:34:54,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=A 2024-11-14T21:34:54,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:54,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=B 2024-11-14T21:34:54,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:54,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=C 2024-11-14T21:34:54,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:54,420 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a9d12804525460eb69a2fbd5bbf512a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731620091912 2024-11-14T21:34:54,423 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:54,449 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114bad282db31bc40ef8f1891352af2f469_2c62143a7911e8dec390d4d4558f8aa4 store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:54,451 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114bad282db31bc40ef8f1891352af2f469_2c62143a7911e8dec390d4d4558f8aa4, store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:54,451 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bad282db31bc40ef8f1891352af2f469_2c62143a7911e8dec390d4d4558f8aa4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:54,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114af1facec8e1b4eab89b2ce1e96093da4_2c62143a7911e8dec390d4d4558f8aa4 is 50, key is test_row_0/A:col10/1731620092726/Put/seqid=0 2024-11-14T21:34:54,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742444_1620 (size=4469) 2024-11-14T21:34:54,488 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c62143a7911e8dec390d4d4558f8aa4#A#compaction#526 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:54,488 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/7f5578842d064e618b351d7320e796a8 is 175, key is test_row_0/A:col10/1731620091932/Put/seqid=0 2024-11-14T21:34:54,492 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c62143a7911e8dec390d4d4558f8aa4#B#compaction#528 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:54,492 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/dc010a382f7d42df9d7d4242309f1039 is 50, key is test_row_0/B:col10/1731620091932/Put/seqid=0 2024-11-14T21:34:54,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742445_1621 (size=12454) 2024-11-14T21:34:54,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:54,529 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114af1facec8e1b4eab89b2ce1e96093da4_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114af1facec8e1b4eab89b2ce1e96093da4_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:54,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/2312e656c7af42dd932156d2aa6e303a, store: [table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:54,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/2312e656c7af42dd932156d2aa6e303a is 175, key is test_row_0/A:col10/1731620092726/Put/seqid=0 2024-11-14T21:34:54,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742446_1622 (size=31835) 2024-11-14T21:34:54,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742447_1623 (size=12881) 2024-11-14T21:34:54,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742448_1624 (size=31255) 2024-11-14T21:34:54,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:54,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620154959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:54,973 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/dc010a382f7d42df9d7d4242309f1039 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/dc010a382f7d42df9d7d4242309f1039 2024-11-14T21:34:54,974 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/7f5578842d064e618b351d7320e796a8 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/7f5578842d064e618b351d7320e796a8 2024-11-14T21:34:54,979 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c62143a7911e8dec390d4d4558f8aa4/B of 2c62143a7911e8dec390d4d4558f8aa4 into dc010a382f7d42df9d7d4242309f1039(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:54,979 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:54,979 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., storeName=2c62143a7911e8dec390d4d4558f8aa4/B, priority=13, startTime=1731620094393; duration=0sec 2024-11-14T21:34:54,979 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:54,979 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:B 2024-11-14T21:34:54,979 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:54,980 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c62143a7911e8dec390d4d4558f8aa4/A of 2c62143a7911e8dec390d4d4558f8aa4 into 7f5578842d064e618b351d7320e796a8(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:54,980 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:54,980 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., storeName=2c62143a7911e8dec390d4d4558f8aa4/A, priority=13, startTime=1731620094393; duration=0sec 2024-11-14T21:34:54,980 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:54,981 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:A 2024-11-14T21:34:54,981 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:54,981 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 2c62143a7911e8dec390d4d4558f8aa4/C is initiating minor compaction (all files) 2024-11-14T21:34:54,982 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c62143a7911e8dec390d4d4558f8aa4/C in TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:54,982 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/244191d1444a47208f33ad9d2477d481, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/3e6df6fc519f4342aca75a6c0a0f59f2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d50702ff4470463dac4498caea46f88a] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp, totalSize=36.2 K 2024-11-14T21:34:54,982 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 244191d1444a47208f33ad9d2477d481, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731620090520 2024-11-14T21:34:54,983 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e6df6fc519f4342aca75a6c0a0f59f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731620090700 2024-11-14T21:34:54,983 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting d50702ff4470463dac4498caea46f88a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731620091912 2024-11-14T21:34:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T21:34:55,001 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/2312e656c7af42dd932156d2aa6e303a 2024-11-14T21:34:55,003 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c62143a7911e8dec390d4d4558f8aa4#C#compaction#529 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:55,004 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/cf7fec4542f24a1daac7ecbc8bbe5b35 is 50, key is test_row_0/C:col10/1731620091932/Put/seqid=0 2024-11-14T21:34:55,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/9da5ab10aa2e4fbdaf84e5cf07173f68 is 50, key is test_row_0/B:col10/1731620092726/Put/seqid=0 2024-11-14T21:34:55,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742449_1625 (size=12881) 2024-11-14T21:34:55,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742450_1626 (size=12301) 2024-11-14T21:34:55,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620155071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:55,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620155290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:55,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:55,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:55,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:55,445 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/cf7fec4542f24a1daac7ecbc8bbe5b35 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/cf7fec4542f24a1daac7ecbc8bbe5b35 2024-11-14T21:34:55,450 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c62143a7911e8dec390d4d4558f8aa4/C of 2c62143a7911e8dec390d4d4558f8aa4 into cf7fec4542f24a1daac7ecbc8bbe5b35(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:55,450 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:55,450 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., storeName=2c62143a7911e8dec390d4d4558f8aa4/C, priority=13, startTime=1731620094393; duration=0sec 2024-11-14T21:34:55,450 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:55,451 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:C 2024-11-14T21:34:55,467 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/9da5ab10aa2e4fbdaf84e5cf07173f68 2024-11-14T21:34:55,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/a1d088ddf83547abb2c71aa6a7e6ac65 is 50, key is test_row_0/C:col10/1731620092726/Put/seqid=0 2024-11-14T21:34:55,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742451_1627 (size=12301) 2024-11-14T21:34:55,542 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/a1d088ddf83547abb2c71aa6a7e6ac65 2024-11-14T21:34:55,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/2312e656c7af42dd932156d2aa6e303a as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/2312e656c7af42dd932156d2aa6e303a 2024-11-14T21:34:55,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620155602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:55,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:55,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:55,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:55,623 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/2312e656c7af42dd932156d2aa6e303a, entries=150, sequenceid=289, filesize=30.5 K 2024-11-14T21:34:55,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/9da5ab10aa2e4fbdaf84e5cf07173f68 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/9da5ab10aa2e4fbdaf84e5cf07173f68 2024-11-14T21:34:55,650 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/9da5ab10aa2e4fbdaf84e5cf07173f68, entries=150, sequenceid=289, filesize=12.0 K 2024-11-14T21:34:55,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/a1d088ddf83547abb2c71aa6a7e6ac65 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/a1d088ddf83547abb2c71aa6a7e6ac65 2024-11-14T21:34:55,688 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/a1d088ddf83547abb2c71aa6a7e6ac65, entries=150, sequenceid=289, filesize=12.0 K 2024-11-14T21:34:55,690 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 2c62143a7911e8dec390d4d4558f8aa4 in 1269ms, sequenceid=289, compaction requested=false 2024-11-14T21:34:55,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:55,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:55,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-11-14T21:34:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-11-14T21:34:55,693 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-14T21:34:55,693 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8570 sec 2024-11-14T21:34:55,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 2.8630 sec 2024-11-14T21:34:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:56,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2c62143a7911e8dec390d4d4558f8aa4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T21:34:56,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=A 2024-11-14T21:34:56,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:56,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=B 2024-11-14T21:34:56,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:56,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=C 2024-11-14T21:34:56,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:56,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111493de2fabf6164180aafbb45a2610df32_2c62143a7911e8dec390d4d4558f8aa4 is 50, key is test_row_0/A:col10/1731620094953/Put/seqid=0 2024-11-14T21:34:56,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742452_1628 (size=12454) 2024-11-14T21:34:56,110 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:56,123 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111493de2fabf6164180aafbb45a2610df32_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111493de2fabf6164180aafbb45a2610df32_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:56,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620156116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:56,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:56,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620156119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:56,125 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,125 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,125 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:56,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620156120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:56,126 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/f8a4358dcc6344f288d418fdfcf9d0cf, store: [table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:56,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620156120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:56,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/f8a4358dcc6344f288d418fdfcf9d0cf is 175, key is test_row_0/A:col10/1731620094953/Put/seqid=0 2024-11-14T21:34:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:56,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620156123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:56,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742453_1629 (size=31255) 2024-11-14T21:34:56,141 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/f8a4358dcc6344f288d418fdfcf9d0cf 2024-11-14T21:34:56,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/a95a5f6a7bc348d89836d0ccbb2a407b is 50, key is test_row_0/B:col10/1731620094953/Put/seqid=0 2024-11-14T21:34:56,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742454_1630 (size=12301) 2024-11-14T21:34:56,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620156231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:56,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:56,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620156231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:56,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:56,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:56,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620156231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:56,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:56,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620156231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:56,380 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T21:34:56,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:56,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620156440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:56,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:56,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620156440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:56,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:56,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:56,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620156440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:56,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620156440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:56,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/a95a5f6a7bc348d89836d0ccbb2a407b 2024-11-14T21:34:56,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/d1a2946c040641dfb9ea6ba6098cab95 is 50, key is test_row_0/C:col10/1731620094953/Put/seqid=0 2024-11-14T21:34:56,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742455_1631 (size=12301) 2024-11-14T21:34:56,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/d1a2946c040641dfb9ea6ba6098cab95 2024-11-14T21:34:56,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/f8a4358dcc6344f288d418fdfcf9d0cf as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/f8a4358dcc6344f288d418fdfcf9d0cf 2024-11-14T21:34:56,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/f8a4358dcc6344f288d418fdfcf9d0cf, entries=150, sequenceid=306, filesize=30.5 K 2024-11-14T21:34:56,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/a95a5f6a7bc348d89836d0ccbb2a407b as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/a95a5f6a7bc348d89836d0ccbb2a407b 2024-11-14T21:34:56,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620156751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:56,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:56,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620156752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:56,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:56,755 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,755 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,755 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:56,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620156753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:56,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620156753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:56,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/a95a5f6a7bc348d89836d0ccbb2a407b, entries=150, sequenceid=306, filesize=12.0 K 2024-11-14T21:34:56,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/d1a2946c040641dfb9ea6ba6098cab95 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d1a2946c040641dfb9ea6ba6098cab95 2024-11-14T21:34:56,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d1a2946c040641dfb9ea6ba6098cab95, entries=150, sequenceid=306, filesize=12.0 K 2024-11-14T21:34:56,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 2c62143a7911e8dec390d4d4558f8aa4 in 747ms, sequenceid=306, compaction requested=true 2024-11-14T21:34:56,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:56,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:34:56,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:56,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:34:56,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:56,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:34:56,814 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:56,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:34:56,814 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:56,815 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:56,815 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:56,815 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 2c62143a7911e8dec390d4d4558f8aa4/B is initiating minor compaction (all files) 2024-11-14T21:34:56,815 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 2c62143a7911e8dec390d4d4558f8aa4/A is initiating minor compaction (all files) 2024-11-14T21:34:56,815 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c62143a7911e8dec390d4d4558f8aa4/B in TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:56,815 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c62143a7911e8dec390d4d4558f8aa4/A in TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:56,815 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/dc010a382f7d42df9d7d4242309f1039, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/9da5ab10aa2e4fbdaf84e5cf07173f68, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/a95a5f6a7bc348d89836d0ccbb2a407b] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp, totalSize=36.6 K 2024-11-14T21:34:56,815 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/7f5578842d064e618b351d7320e796a8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/2312e656c7af42dd932156d2aa6e303a, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/f8a4358dcc6344f288d418fdfcf9d0cf] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp, totalSize=92.1 K 2024-11-14T21:34:56,815 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:56,815 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. files: [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/7f5578842d064e618b351d7320e796a8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/2312e656c7af42dd932156d2aa6e303a, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/f8a4358dcc6344f288d418fdfcf9d0cf] 2024-11-14T21:34:56,818 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc010a382f7d42df9d7d4242309f1039, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731620091912 2024-11-14T21:34:56,818 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f5578842d064e618b351d7320e796a8, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731620091912 2024-11-14T21:34:56,819 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 2312e656c7af42dd932156d2aa6e303a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731620092706 2024-11-14T21:34:56,819 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9da5ab10aa2e4fbdaf84e5cf07173f68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731620092706 2024-11-14T21:34:56,821 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting f8a4358dcc6344f288d418fdfcf9d0cf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731620094927 2024-11-14T21:34:56,821 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting a95a5f6a7bc348d89836d0ccbb2a407b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731620094927 2024-11-14T21:34:56,841 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:56,845 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c62143a7911e8dec390d4d4558f8aa4#B#compaction#536 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:56,845 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/65c0e50197eb499e94e2f7e0d532dd12 is 50, key is test_row_0/B:col10/1731620094953/Put/seqid=0 2024-11-14T21:34:56,855 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111446f7d854afce45d6b0c10aefd3791212_2c62143a7911e8dec390d4d4558f8aa4 store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:56,857 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111446f7d854afce45d6b0c10aefd3791212_2c62143a7911e8dec390d4d4558f8aa4, store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:56,857 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111446f7d854afce45d6b0c10aefd3791212_2c62143a7911e8dec390d4d4558f8aa4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:56,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742457_1633 (size=4469) 2024-11-14T21:34:56,914 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c62143a7911e8dec390d4d4558f8aa4#A#compaction#535 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:56,914 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/8421c7c265044c09b8ccc6e0a53e646c is 175, key is test_row_0/A:col10/1731620094953/Put/seqid=0 2024-11-14T21:34:56,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742456_1632 (size=12983) 2024-11-14T21:34:56,942 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/65c0e50197eb499e94e2f7e0d532dd12 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/65c0e50197eb499e94e2f7e0d532dd12 2024-11-14T21:34:56,946 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c62143a7911e8dec390d4d4558f8aa4/B of 2c62143a7911e8dec390d4d4558f8aa4 into 65c0e50197eb499e94e2f7e0d532dd12(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:56,946 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:56,946 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., storeName=2c62143a7911e8dec390d4d4558f8aa4/B, priority=13, startTime=1731620096814; duration=0sec 2024-11-14T21:34:56,946 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:56,946 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:B 2024-11-14T21:34:56,946 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:56,947 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:56,947 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 2c62143a7911e8dec390d4d4558f8aa4/C is initiating minor compaction (all files) 2024-11-14T21:34:56,947 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c62143a7911e8dec390d4d4558f8aa4/C in TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:56,948 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/cf7fec4542f24a1daac7ecbc8bbe5b35, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/a1d088ddf83547abb2c71aa6a7e6ac65, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d1a2946c040641dfb9ea6ba6098cab95] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp, totalSize=36.6 K 2024-11-14T21:34:56,948 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf7fec4542f24a1daac7ecbc8bbe5b35, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731620091912 2024-11-14T21:34:56,948 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1d088ddf83547abb2c71aa6a7e6ac65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731620092706 2024-11-14T21:34:56,949 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1a2946c040641dfb9ea6ba6098cab95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731620094927 2024-11-14T21:34:56,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742458_1634 (size=31937) 2024-11-14T21:34:56,961 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c62143a7911e8dec390d4d4558f8aa4#C#compaction#537 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:56,962 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/5a759fcd3d5f469b869eb4ff20a4ae42 is 50, key is test_row_0/C:col10/1731620094953/Put/seqid=0 2024-11-14T21:34:57,010 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/8421c7c265044c09b8ccc6e0a53e646c as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/8421c7c265044c09b8ccc6e0a53e646c 2024-11-14T21:34:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T21:34:57,011 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:34:57,012 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:34:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-14T21:34:57,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742459_1635 (size=12983) 2024-11-14T21:34:57,016 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:34:57,018 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:34:57,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:34:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T21:34:57,041 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c62143a7911e8dec390d4d4558f8aa4/A of 2c62143a7911e8dec390d4d4558f8aa4 into 8421c7c265044c09b8ccc6e0a53e646c(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:57,041 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:57,041 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., storeName=2c62143a7911e8dec390d4d4558f8aa4/A, priority=13, startTime=1731620096814; duration=0sec 2024-11-14T21:34:57,042 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:57,042 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:A 2024-11-14T21:34:57,051 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/5a759fcd3d5f469b869eb4ff20a4ae42 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/5a759fcd3d5f469b869eb4ff20a4ae42 2024-11-14T21:34:57,075 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c62143a7911e8dec390d4d4558f8aa4/C of 2c62143a7911e8dec390d4d4558f8aa4 into 5a759fcd3d5f469b869eb4ff20a4ae42(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:34:57,075 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:57,075 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., storeName=2c62143a7911e8dec390d4d4558f8aa4/C, priority=13, startTime=1731620096814; duration=0sec 2024-11-14T21:34:57,075 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:57,075 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:C 2024-11-14T21:34:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T21:34:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:57,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2c62143a7911e8dec390d4d4558f8aa4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T21:34:57,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=A 2024-11-14T21:34:57,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:57,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=B 2024-11-14T21:34:57,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:57,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=C 2024-11-14T21:34:57,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:57,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T21:34:57,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:57,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:57,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:57,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:57,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:57,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111402dc5c014675412cb1af0683d3dbd920_2c62143a7911e8dec390d4d4558f8aa4 is 50, key is test_row_0/A:col10/1731620096118/Put/seqid=0 2024-11-14T21:34:57,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742460_1636 (size=14994) 2024-11-14T21:34:57,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620157260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:57,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620157261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:57,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620157261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:57,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:57,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:57,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:57,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:57,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:57,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:57,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:57,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:57,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:57,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620157264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:57,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620157271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:57,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:57,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:57,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:57,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T21:34:57,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:57,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:57,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:57,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:57,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T21:34:57,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620157381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:57,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:57,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:57,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:57,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T21:34:57,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:57,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:57,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:57,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:57,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:57,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620157593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:57,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:57,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:57,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:57,627 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:57,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T21:34:57,653 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111402dc5c014675412cb1af0683d3dbd920_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111402dc5c014675412cb1af0683d3dbd920_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:57,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T21:34:57,656 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/5117528cb1694a96a73b30df57583146, store: [table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:57,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:57,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:57,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:57,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:57,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:57,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/5117528cb1694a96a73b30df57583146 is 175, key is test_row_0/A:col10/1731620096118/Put/seqid=0 2024-11-14T21:34:57,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:57,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742461_1637 (size=39949) 2024-11-14T21:34:57,698 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/5117528cb1694a96a73b30df57583146 2024-11-14T21:34:57,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/c423d3da07f64871802ffc75b808a643 is 50, key is test_row_0/B:col10/1731620096118/Put/seqid=0 2024-11-14T21:34:57,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742462_1638 (size=12301) 2024-11-14T21:34:57,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/c423d3da07f64871802ffc75b808a643 2024-11-14T21:34:57,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/6cf04d529a61429fa6542108479fb824 is 50, key is test_row_0/C:col10/1731620096118/Put/seqid=0 2024-11-14T21:34:57,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T21:34:57,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:57,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:57,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:57,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:57,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:57,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:57,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742463_1639 (size=12301) 2024-11-14T21:34:57,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:57,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620157911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:57,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:57,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:57,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:57,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T21:34:57,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:57,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:57,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:57,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:57,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:57,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:58,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T21:34:58,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:58,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:58,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:58,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:58,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:58,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:58,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T21:34:58,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/6cf04d529a61429fa6542108479fb824 2024-11-14T21:34:58,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620158271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:58,272 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_1' in 2c62143a7911e8dec390d4d4558f8aa4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:34:58,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:58,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:58,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:58,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620158281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:58,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:58,283 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_0' in 2c62143a7911e8dec390d4d4558f8aa4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:34:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620158281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:58,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/5117528cb1694a96a73b30df57583146 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/5117528cb1694a96a73b30df57583146 2024-11-14T21:34:58,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:58,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:58,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:58,284 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_1' in 2c62143a7911e8dec390d4d4558f8aa4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:34:58,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:58,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:58,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:58,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620158282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:58,285 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_2' in 2c62143a7911e8dec390d4d4558f8aa4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:34:58,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:58,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:58,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:58,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T21:34:58,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:58,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:58,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:58,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:58,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:58,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/5117528cb1694a96a73b30df57583146, entries=200, sequenceid=328, filesize=39.0 K 2024-11-14T21:34:58,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/c423d3da07f64871802ffc75b808a643 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/c423d3da07f64871802ffc75b808a643 2024-11-14T21:34:58,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/c423d3da07f64871802ffc75b808a643, entries=150, sequenceid=328, filesize=12.0 K 2024-11-14T21:34:58,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/6cf04d529a61429fa6542108479fb824 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/6cf04d529a61429fa6542108479fb824 2024-11-14T21:34:58,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/6cf04d529a61429fa6542108479fb824, entries=150, sequenceid=328, filesize=12.0 K 2024-11-14T21:34:58,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 2c62143a7911e8dec390d4d4558f8aa4 in 1167ms, sequenceid=328, compaction requested=false 2024-11-14T21:34:58,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:58,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2c62143a7911e8dec390d4d4558f8aa4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T21:34:58,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=A 2024-11-14T21:34:58,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:58,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=B 2024-11-14T21:34:58,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:58,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=C 2024-11-14T21:34:58,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:58,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111437c814af297f43a388f2d1ffb18c304b_2c62143a7911e8dec390d4d4558f8aa4 is 50, key is test_row_0/A:col10/1731620097262/Put/seqid=0 2024-11-14T21:34:58,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T21:34:58,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:58,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:58,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:58,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:58,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:58,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742464_1640 (size=12454) 2024-11-14T21:34:58,504 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:34:58,521 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111437c814af297f43a388f2d1ffb18c304b_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111437c814af297f43a388f2d1ffb18c304b_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:34:58,529 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/df94a3f1cfbc4c72b9a5076612a270e7, store: [table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:58,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/df94a3f1cfbc4c72b9a5076612a270e7 is 175, key is test_row_0/A:col10/1731620097262/Put/seqid=0 2024-11-14T21:34:58,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:58,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620158545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:58,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:58,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:58,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:58,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742465_1641 (size=31255) 2024-11-14T21:34:58,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T21:34:58,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:58,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:58,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:58,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:58,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:58,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620158661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:58,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T21:34:58,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:58,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:58,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:58,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:58,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:58,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620158871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:58,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:58,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:58,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:58,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T21:34:58,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:58,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:58,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:58,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:58,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:58,975 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/df94a3f1cfbc4c72b9a5076612a270e7 2024-11-14T21:34:59,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/2ec507b9e8814f9cb09942db95157597 is 50, key is test_row_0/B:col10/1731620097262/Put/seqid=0 2024-11-14T21:34:59,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T21:34:59,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:59,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:59,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:59,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:59,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:59,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:59,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742466_1642 (size=12301) 2024-11-14T21:34:59,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/2ec507b9e8814f9cb09942db95157597 2024-11-14T21:34:59,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/4438305d86fb4c79be7027b4f0056353 is 50, key is test_row_0/C:col10/1731620097262/Put/seqid=0 2024-11-14T21:34:59,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T21:34:59,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:59,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620159182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:59,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742467_1643 (size=12301) 2024-11-14T21:34:59,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T21:34:59,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:59,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:59,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:59,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:59,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:59,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:59,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T21:34:59,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:59,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:59,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:59,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:59,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:59,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T21:34:59,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:59,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:34:59,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:59,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:59,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:34:59,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/4438305d86fb4c79be7027b4f0056353 2024-11-14T21:34:59,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/df94a3f1cfbc4c72b9a5076612a270e7 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/df94a3f1cfbc4c72b9a5076612a270e7 2024-11-14T21:34:59,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/df94a3f1cfbc4c72b9a5076612a270e7, entries=150, sequenceid=342, filesize=30.5 K 2024-11-14T21:34:59,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/2ec507b9e8814f9cb09942db95157597 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/2ec507b9e8814f9cb09942db95157597 2024-11-14T21:34:59,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:34:59,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620159690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:34:59,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:59,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:34:59,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:34:59,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/2ec507b9e8814f9cb09942db95157597, entries=150, sequenceid=342, filesize=12.0 K 2024-11-14T21:34:59,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/4438305d86fb4c79be7027b4f0056353 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/4438305d86fb4c79be7027b4f0056353 2024-11-14T21:34:59,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/4438305d86fb4c79be7027b4f0056353, entries=150, sequenceid=342, filesize=12.0 K 2024-11-14T21:34:59,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T21:34:59,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 2c62143a7911e8dec390d4d4558f8aa4 in 1313ms, sequenceid=342, compaction requested=true 2024-11-14T21:34:59,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:34:59,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:34:59,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:34:59,740 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:59,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:34:59,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:34:59,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:34:59,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T21:34:59,740 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:34:59,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:59,742 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing 2c62143a7911e8dec390d4d4558f8aa4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T21:34:59,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=A 2024-11-14T21:34:59,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:59,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=B 2024-11-14T21:34:59,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:59,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=C 2024-11-14T21:34:59,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:34:59,747 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:59,747 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 2c62143a7911e8dec390d4d4558f8aa4/B is initiating minor compaction (all files) 2024-11-14T21:34:59,747 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c62143a7911e8dec390d4d4558f8aa4/B in TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:59,747 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/65c0e50197eb499e94e2f7e0d532dd12, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/c423d3da07f64871802ffc75b808a643, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/2ec507b9e8814f9cb09942db95157597] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp, totalSize=36.7 K 2024-11-14T21:34:59,747 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:34:59,747 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1541): 2c62143a7911e8dec390d4d4558f8aa4/A is initiating minor compaction (all files) 2024-11-14T21:34:59,747 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c62143a7911e8dec390d4d4558f8aa4/A in TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:59,747 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/8421c7c265044c09b8ccc6e0a53e646c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/5117528cb1694a96a73b30df57583146, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/df94a3f1cfbc4c72b9a5076612a270e7] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp, totalSize=100.7 K 2024-11-14T21:34:59,747 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:34:59,747 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. files: [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/8421c7c265044c09b8ccc6e0a53e646c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/5117528cb1694a96a73b30df57583146, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/df94a3f1cfbc4c72b9a5076612a270e7] 2024-11-14T21:34:59,751 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 65c0e50197eb499e94e2f7e0d532dd12, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731620094927 2024-11-14T21:34:59,751 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8421c7c265044c09b8ccc6e0a53e646c, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731620094927 2024-11-14T21:34:59,755 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting c423d3da07f64871802ffc75b808a643, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731620096117 2024-11-14T21:34:59,755 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5117528cb1694a96a73b30df57583146, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731620096117 2024-11-14T21:34:59,759 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ec507b9e8814f9cb09942db95157597, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731620097235 2024-11-14T21:34:59,759 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] compactions.Compactor(225): Compacting df94a3f1cfbc4c72b9a5076612a270e7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731620097235 2024-11-14T21:34:59,792 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c62143a7911e8dec390d4d4558f8aa4#B#compaction#544 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:59,793 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/905e5723d52345939c83e9ac02916a58 is 50, key is test_row_0/B:col10/1731620097262/Put/seqid=0 2024-11-14T21:34:59,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141ac0ec7b998343b5a29ea6feb60e3245_2c62143a7911e8dec390d4d4558f8aa4 is 50, key is test_row_0/A:col10/1731620098543/Put/seqid=0 2024-11-14T21:34:59,809 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:59,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742469_1645 (size=12454) 2024-11-14T21:34:59,827 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111469a6100d02f8440b84a7f3db390aee5d_2c62143a7911e8dec390d4d4558f8aa4 store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:59,835 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111469a6100d02f8440b84a7f3db390aee5d_2c62143a7911e8dec390d4d4558f8aa4, store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:59,835 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111469a6100d02f8440b84a7f3db390aee5d_2c62143a7911e8dec390d4d4558f8aa4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:34:59,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742468_1644 (size=13085) 2024-11-14T21:34:59,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742470_1646 (size=4469) 2024-11-14T21:34:59,880 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c62143a7911e8dec390d4d4558f8aa4#A#compaction#546 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:34:59,880 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/734904b958db4e1c88adc37f68d30159 is 175, key is test_row_0/A:col10/1731620097262/Put/seqid=0 2024-11-14T21:34:59,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742471_1647 (size=32039) 2024-11-14T21:35:00,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:35:00,250 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141ac0ec7b998343b5a29ea6feb60e3245_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141ac0ec7b998343b5a29ea6feb60e3245_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:00,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/14685a1c507e433a9891da537d1e33da, store: [table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:35:00,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/14685a1c507e433a9891da537d1e33da is 175, key is test_row_0/A:col10/1731620098543/Put/seqid=0 2024-11-14T21:35:00,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742472_1648 (size=31255) 2024-11-14T21:35:00,279 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/14685a1c507e433a9891da537d1e33da 2024-11-14T21:35:00,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:00,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:35:00,294 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/905e5723d52345939c83e9ac02916a58 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/905e5723d52345939c83e9ac02916a58 2024-11-14T21:35:00,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/75b9e3c4979d49beac8291a7edceee39 is 50, key is test_row_0/B:col10/1731620098543/Put/seqid=0 2024-11-14T21:35:00,337 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c62143a7911e8dec390d4d4558f8aa4/B of 2c62143a7911e8dec390d4d4558f8aa4 into 905e5723d52345939c83e9ac02916a58(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:35:00,337 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:35:00,337 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., storeName=2c62143a7911e8dec390d4d4558f8aa4/B, priority=13, startTime=1731620099740; duration=0sec 2024-11-14T21:35:00,337 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:35:00,337 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:B 2024-11-14T21:35:00,337 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T21:35:00,338 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/734904b958db4e1c88adc37f68d30159 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/734904b958db4e1c88adc37f68d30159 2024-11-14T21:35:00,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742473_1649 (size=12301) 2024-11-14T21:35:00,348 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T21:35:00,348 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1541): 2c62143a7911e8dec390d4d4558f8aa4/C is initiating minor compaction (all files) 2024-11-14T21:35:00,348 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c62143a7911e8dec390d4d4558f8aa4/C in TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:35:00,348 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/5a759fcd3d5f469b869eb4ff20a4ae42, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/6cf04d529a61429fa6542108479fb824, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/4438305d86fb4c79be7027b4f0056353] into tmpdir=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp, totalSize=36.7 K 2024-11-14T21:35:00,351 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a759fcd3d5f469b869eb4ff20a4ae42, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731620094927 2024-11-14T21:35:00,353 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cf04d529a61429fa6542108479fb824, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731620096117 2024-11-14T21:35:00,354 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] compactions.Compactor(225): Compacting 4438305d86fb4c79be7027b4f0056353, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731620097235 2024-11-14T21:35:00,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620160342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:00,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:00,360 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c62143a7911e8dec390d4d4558f8aa4/A of 2c62143a7911e8dec390d4d4558f8aa4 into 734904b958db4e1c88adc37f68d30159(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:35:00,360 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:35:00,360 INFO [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., storeName=2c62143a7911e8dec390d4d4558f8aa4/A, priority=13, startTime=1731620099740; duration=0sec 2024-11-14T21:35:00,360 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:35:00,360 DEBUG [RS:0;ef788a4cade8:37683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:A 2024-11-14T21:35:00,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620160347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:00,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620160348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:00,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620160359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:00,404 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c62143a7911e8dec390d4d4558f8aa4#C#compaction#548 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:35:00,405 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/bf536083571243b4b5fc3e3c2231def4 is 50, key is test_row_0/C:col10/1731620097262/Put/seqid=0 2024-11-14T21:35:00,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620160461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:00,468 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,468 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,468 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:00,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742474_1650 (size=13085) 2024-11-14T21:35:00,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620160471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:00,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:00,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620160471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620160471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:00,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:00,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:00,481 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/bf536083571243b4b5fc3e3c2231def4 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/bf536083571243b4b5fc3e3c2231def4 2024-11-14T21:35:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:00,489 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c62143a7911e8dec390d4d4558f8aa4/C of 2c62143a7911e8dec390d4d4558f8aa4 into bf536083571243b4b5fc3e3c2231def4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:35:00,489 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:35:00,489 INFO [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., storeName=2c62143a7911e8dec390d4d4558f8aa4/C, priority=13, startTime=1731620099740; duration=0sec 2024-11-14T21:35:00,489 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:35:00,489 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:C 2024-11-14T21:35:00,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620160680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:00,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620160680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:00,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:00,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:00,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620160691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:00,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:00,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620160691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:00,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:00,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:00,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42368 deadline: 1731620160700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:00,703 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ef788a4cade8,37683,1731619918560 for 'test_row_0' in 2c62143a7911e8dec390d4d4558f8aa4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T21:35:00,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:00,748 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/75b9e3c4979d49beac8291a7edceee39 2024-11-14T21:35:00,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/f1e19067bed14dd28413071b656ed1f3 is 50, key is test_row_0/C:col10/1731620098543/Put/seqid=0 2024-11-14T21:35:00,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742475_1651 (size=12301) 2024-11-14T21:35:00,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620160990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:00,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620160990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:00,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:00,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:00,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:01,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620161000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:01,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620161001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:01,160 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/f1e19067bed14dd28413071b656ed1f3 2024-11-14T21:35:01,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/14685a1c507e433a9891da537d1e33da as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/14685a1c507e433a9891da537d1e33da 2024-11-14T21:35:01,174 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/14685a1c507e433a9891da537d1e33da, entries=150, sequenceid=360, filesize=30.5 K 2024-11-14T21:35:01,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/75b9e3c4979d49beac8291a7edceee39 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/75b9e3c4979d49beac8291a7edceee39 2024-11-14T21:35:01,180 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/75b9e3c4979d49beac8291a7edceee39, entries=150, sequenceid=360, filesize=12.0 K 2024-11-14T21:35:01,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/f1e19067bed14dd28413071b656ed1f3 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/f1e19067bed14dd28413071b656ed1f3 2024-11-14T21:35:01,189 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/f1e19067bed14dd28413071b656ed1f3, entries=150, sequenceid=360, filesize=12.0 K 2024-11-14T21:35:01,191 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2c62143a7911e8dec390d4d4558f8aa4 in 1449ms, sequenceid=360, compaction requested=false 2024-11-14T21:35:01,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:35:01,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:35:01,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-14T21:35:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-14T21:35:01,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-14T21:35:01,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.1770 sec 2024-11-14T21:35:01,199 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 4.1860 sec 2024-11-14T21:35:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T21:35:01,202 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:35:01,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T21:35:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-14T21:35:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T21:35:01,205 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T21:35:01,205 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T21:35:01,205 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T21:35:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T21:35:01,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T21:35:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:35:01,359 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 2c62143a7911e8dec390d4d4558f8aa4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T21:35:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=A 2024-11-14T21:35:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:35:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=B 2024-11-14T21:35:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:35:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=C 2024-11-14T21:35:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:35:01,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114046c65f6dd42484f9b6243eeb70ef438_2c62143a7911e8dec390d4d4558f8aa4 is 50, key is test_row_0/A:col10/1731620100331/Put/seqid=0 2024-11-14T21:35:01,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742476_1652 (size=12454) 2024-11-14T21:35:01,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:01,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. as already flushing 2024-11-14T21:35:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T21:35:01,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620161537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:01,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620161552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620161556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:01,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620161560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:01,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620161661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:01,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620161671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:01,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:01,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620161671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:01,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:01,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620161671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:01,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:01,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:35:01,796 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114046c65f6dd42484f9b6243eeb70ef438_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114046c65f6dd42484f9b6243eeb70ef438_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:01,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/6414e4ba68854dbb8a6075aa095d7479, store: [table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:35:01,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/6414e4ba68854dbb8a6075aa095d7479 is 175, key is test_row_0/A:col10/1731620100331/Put/seqid=0 2024-11-14T21:35:01,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742477_1653 (size=31255) 2024-11-14T21:35:01,819 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=377, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/6414e4ba68854dbb8a6075aa095d7479 2024-11-14T21:35:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T21:35:01,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/51716121fc0c49f3b8dc16d1727bc290 is 50, key is test_row_0/B:col10/1731620100331/Put/seqid=0 2024-11-14T21:35:01,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742478_1654 (size=12301) 2024-11-14T21:35:01,855 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/51716121fc0c49f3b8dc16d1727bc290 2024-11-14T21:35:01,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/d754475c722b40bf8fd3aad151a41ac2 is 50, key is test_row_0/C:col10/1731620100331/Put/seqid=0 2024-11-14T21:35:01,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:01,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1731620161870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:01,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:01,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42376 deadline: 1731620161882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:01,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42416 deadline: 1731620161882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:01,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742479_1655 (size=12301) 2024-11-14T21:35:01,899 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/d754475c722b40bf8fd3aad151a41ac2 2024-11-14T21:35:01,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:35:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42426 deadline: 1731620161895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:01,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 , the old value is region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:35:01,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., hostname=ef788a4cade8,37683,1731619918560, seqNum=5 because the exception is null or not the one we care about 2024-11-14T21:35:01,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/6414e4ba68854dbb8a6075aa095d7479 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/6414e4ba68854dbb8a6075aa095d7479 2024-11-14T21:35:01,913 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/6414e4ba68854dbb8a6075aa095d7479, entries=150, sequenceid=377, filesize=30.5 K 2024-11-14T21:35:01,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/51716121fc0c49f3b8dc16d1727bc290 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/51716121fc0c49f3b8dc16d1727bc290 2024-11-14T21:35:01,918 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/51716121fc0c49f3b8dc16d1727bc290, entries=150, sequenceid=377, filesize=12.0 K 2024-11-14T21:35:01,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/d754475c722b40bf8fd3aad151a41ac2 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d754475c722b40bf8fd3aad151a41ac2 2024-11-14T21:35:01,921 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d754475c722b40bf8fd3aad151a41ac2, entries=150, sequenceid=377, filesize=12.0 K 2024-11-14T21:35:01,922 INFO [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 2c62143a7911e8dec390d4d4558f8aa4 in 563ms, sequenceid=377, compaction requested=true 2024-11-14T21:35:01,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:35:01,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:35:01,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-14T21:35:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-14T21:35:01,925 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-14T21:35:01,925 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 718 msec 2024-11-14T21:35:01,926 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 723 msec 2024-11-14T21:35:02,165 INFO [Thread-2471 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2471. 2024-11-14T21:35:02,165 DEBUG [Thread-2471 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:35:02,165 DEBUG [Thread-2471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:02,165 INFO [Thread-2475 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2475. 2024-11-14T21:35:02,165 DEBUG [Thread-2475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:35:02,165 DEBUG [Thread-2471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:02,165 DEBUG [Thread-2475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:02,165 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:35:02,165 INFO [Thread-2473 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2473. 2024-11-14T21:35:02,165 DEBUG [Thread-2473 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:35:02,165 DEBUG [Thread-2473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:02,165 DEBUG [Thread-2473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:02,165 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:35:02,166 DEBUG [Thread-2475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:02,166 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:35:02,168 INFO [Thread-2469 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2469. 2024-11-14T21:35:02,168 DEBUG [Thread-2469 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:35:02,168 DEBUG [Thread-2469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:02,168 DEBUG [Thread-2469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:02,168 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:35:02,168 INFO [Thread-2467 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2467. 2024-11-14T21:35:02,168 DEBUG [Thread-2467 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:35:02,168 DEBUG [Thread-2467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:02,168 DEBUG [Thread-2467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:02,168 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:35:02,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37683 {}] regionserver.HRegion(8855): Flush requested on 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:02,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2c62143a7911e8dec390d4d4558f8aa4 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-14T21:35:02,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=A 2024-11-14T21:35:02,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:35:02,193 INFO [Thread-2462 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2462. 2024-11-14T21:35:02,193 INFO [Thread-2456 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2456. 2024-11-14T21:35:02,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=B 2024-11-14T21:35:02,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:35:02,193 DEBUG [Thread-2462 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:35:02,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=C 2024-11-14T21:35:02,193 DEBUG [Thread-2456 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:35:02,193 DEBUG [Thread-2462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:02,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T21:35:02,193 DEBUG [Thread-2456 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:02,193 DEBUG [Thread-2462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:02,193 DEBUG [Thread-2456 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:02,193 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:35:02,194 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:35:02,194 INFO [Thread-2460 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2460. 2024-11-14T21:35:02,194 DEBUG [Thread-2460 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:35:02,194 DEBUG [Thread-2460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:02,194 DEBUG [Thread-2460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:02,194 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:35:02,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142588a2b0782b4ac092e655ee11694ada_2c62143a7911e8dec390d4d4558f8aa4 is 50, key is test_row_0/A:col10/1731620102191/Put/seqid=0 2024-11-14T21:35:02,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742480_1656 (size=12454) 2024-11-14T21:35:02,212 INFO [Thread-2464 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2464. 2024-11-14T21:35:02,212 DEBUG [Thread-2464 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:35:02,212 DEBUG [Thread-2464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:02,212 DEBUG [Thread-2464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:02,212 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:35:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T21:35:02,341 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:35:02,602 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:35:02,606 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142588a2b0782b4ac092e655ee11694ada_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142588a2b0782b4ac092e655ee11694ada_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:02,607 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/1d116a861931444785ff6a49d143f5e7, store: [table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:35:02,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/1d116a861931444785ff6a49d143f5e7 is 175, key is test_row_0/A:col10/1731620102191/Put/seqid=0 2024-11-14T21:35:02,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742481_1657 (size=31255) 2024-11-14T21:35:02,616 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=399, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/1d116a861931444785ff6a49d143f5e7 2024-11-14T21:35:02,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/0fb1123b5e4a438299451cddf90391a6 is 50, key is test_row_0/B:col10/1731620102191/Put/seqid=0 2024-11-14T21:35:02,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742482_1658 (size=12301) 2024-11-14T21:35:02,723 INFO [Thread-2458 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2458. 2024-11-14T21:35:02,723 DEBUG [Thread-2458 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T21:35:02,723 DEBUG [Thread-2458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:02,723 DEBUG [Thread-2458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:02,723 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:35:02,723 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T21:35:02,723 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-14T21:35:02,723 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 149 2024-11-14T21:35:02,723 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-14T21:35:02,723 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-14T21:35:02,723 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-14T21:35:02,723 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T21:35:02,723 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2043 2024-11-14T21:35:02,723 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2094 2024-11-14T21:35:02,723 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2077 2024-11-14T21:35:02,723 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2093 2024-11-14T21:35:02,723 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2071 2024-11-14T21:35:02,723 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T21:35:02,723 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T21:35:02,724 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T21:35:02,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:02,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:02,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:35:02,724 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T21:35:02,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T21:35:02,728 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731620102728"}]},"ts":"1731620102728"} 2024-11-14T21:35:02,730 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T21:35:02,730 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T21:35:02,730 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T21:35:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T21:35:02,732 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c62143a7911e8dec390d4d4558f8aa4, UNASSIGN}] 2024-11-14T21:35:02,733 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=148, ppid=147, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c62143a7911e8dec390d4d4558f8aa4, UNASSIGN 2024-11-14T21:35:02,734 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=148 updating hbase:meta row=2c62143a7911e8dec390d4d4558f8aa4, regionState=CLOSING, regionLocation=ef788a4cade8,37683,1731619918560 2024-11-14T21:35:02,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c62143a7911e8dec390d4d4558f8aa4, UNASSIGN because future has completed 2024-11-14T21:35:02,736 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T21:35:02,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560}] 2024-11-14T21:35:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T21:35:02,888 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] handler.UnassignRegionHandler(122): Close 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:02,888 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T21:35:02,888 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(1722): Closing 2c62143a7911e8dec390d4d4558f8aa4, disabling compactions & flushes 2024-11-14T21:35:02,888 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:35:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T21:35:03,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/0fb1123b5e4a438299451cddf90391a6 2024-11-14T21:35:03,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/e72a04d3dccd4820b4932c235b33f48a is 50, key is test_row_0/C:col10/1731620102191/Put/seqid=0 2024-11-14T21:35:03,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742483_1659 (size=12301) 2024-11-14T21:35:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T21:35:03,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/e72a04d3dccd4820b4932c235b33f48a 2024-11-14T21:35:03,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/1d116a861931444785ff6a49d143f5e7 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/1d116a861931444785ff6a49d143f5e7 2024-11-14T21:35:03,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/1d116a861931444785ff6a49d143f5e7, entries=150, sequenceid=399, filesize=30.5 K 2024-11-14T21:35:03,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/0fb1123b5e4a438299451cddf90391a6 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/0fb1123b5e4a438299451cddf90391a6 2024-11-14T21:35:03,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/0fb1123b5e4a438299451cddf90391a6, entries=150, sequenceid=399, filesize=12.0 K 2024-11-14T21:35:03,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/e72a04d3dccd4820b4932c235b33f48a as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/e72a04d3dccd4820b4932c235b33f48a 2024-11-14T21:35:03,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/e72a04d3dccd4820b4932c235b33f48a, entries=150, sequenceid=399, filesize=12.0 K 2024-11-14T21:35:03,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=13.42 KB/13740 for 2c62143a7911e8dec390d4d4558f8aa4 in 1288ms, sequenceid=399, compaction requested=true 2024-11-14T21:35:03,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2c62143a7911e8dec390d4d4558f8aa4: 2024-11-14T21:35:03,481 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:35:03,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T21:35:03,481 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:35:03,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T21:35:03,481 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. because compaction request was cancelled 2024-11-14T21:35:03,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T21:35:03,481 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. after waiting 0 ms 2024-11-14T21:35:03,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:35:03,481 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:A 2024-11-14T21:35:03,481 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:35:03,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c62143a7911e8dec390d4d4558f8aa4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T21:35:03,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T21:35:03,481 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. because compaction request was cancelled 2024-11-14T21:35:03,481 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:B 2024-11-14T21:35:03,481 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. because compaction request was cancelled 2024-11-14T21:35:03,481 DEBUG [RS:0;ef788a4cade8:37683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c62143a7911e8dec390d4d4558f8aa4:C 2024-11-14T21:35:03,481 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(2902): Flushing 2c62143a7911e8dec390d4d4558f8aa4 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-14T21:35:03,481 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=A 2024-11-14T21:35:03,481 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:35:03,482 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=B 2024-11-14T21:35:03,482 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:35:03,482 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2c62143a7911e8dec390d4d4558f8aa4, store=C 2024-11-14T21:35:03,482 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T21:35:03,491 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111477d5304c62c1454daaa5d9351775abd9_2c62143a7911e8dec390d4d4558f8aa4 is 50, key is test_row_0/A:col10/1731620102721/Put/seqid=0 2024-11-14T21:35:03,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742484_1660 (size=7374) 2024-11-14T21:35:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T21:35:03,901 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:35:03,904 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111477d5304c62c1454daaa5d9351775abd9_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111477d5304c62c1454daaa5d9351775abd9_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:03,905 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/0f59b33dbb5743468715192712a67728, store: [table=TestAcidGuarantees family=A region=2c62143a7911e8dec390d4d4558f8aa4] 2024-11-14T21:35:03,905 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/0f59b33dbb5743468715192712a67728 is 175, key is test_row_0/A:col10/1731620102721/Put/seqid=0 2024-11-14T21:35:03,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742485_1661 (size=13865) 2024-11-14T21:35:04,309 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=404, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/0f59b33dbb5743468715192712a67728 2024-11-14T21:35:04,314 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/6e90a94828d14b1399bc9ba632e578a0 is 50, key is test_row_0/B:col10/1731620102721/Put/seqid=0 2024-11-14T21:35:04,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742486_1662 (size=7415) 2024-11-14T21:35:04,719 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/6e90a94828d14b1399bc9ba632e578a0 2024-11-14T21:35:04,726 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/3201afa336ca4fca9577ebcea5e7aeff is 50, key is test_row_0/C:col10/1731620102721/Put/seqid=0 2024-11-14T21:35:04,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742487_1663 (size=7415) 2024-11-14T21:35:04,747 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/3201afa336ca4fca9577ebcea5e7aeff 2024-11-14T21:35:04,753 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/A/0f59b33dbb5743468715192712a67728 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/0f59b33dbb5743468715192712a67728 2024-11-14T21:35:04,756 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/0f59b33dbb5743468715192712a67728, entries=50, sequenceid=404, filesize=13.5 K 2024-11-14T21:35:04,757 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/B/6e90a94828d14b1399bc9ba632e578a0 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/6e90a94828d14b1399bc9ba632e578a0 2024-11-14T21:35:04,760 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/6e90a94828d14b1399bc9ba632e578a0, entries=50, sequenceid=404, filesize=7.2 K 2024-11-14T21:35:04,760 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/.tmp/C/3201afa336ca4fca9577ebcea5e7aeff as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/3201afa336ca4fca9577ebcea5e7aeff 2024-11-14T21:35:04,764 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/3201afa336ca4fca9577ebcea5e7aeff, entries=50, sequenceid=404, filesize=7.2 K 2024-11-14T21:35:04,765 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 2c62143a7911e8dec390d4d4558f8aa4 in 1283ms, sequenceid=404, compaction requested=true 2024-11-14T21:35:04,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/545447f6859f4e83b3e13800a91b6ef2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/35da1178ea5445baa4787629c8a7aac1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ae1a92eb47c4452da72d7112a7eb2e30, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/1ad4548083d6477c8281289276e1a3e1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/9784e0d35d9c4ae4b533647328997a1d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/59eb90c1efca4d2e8cc347ccda32bbfb, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/54da3f03705f4caf82361fb01400953f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/56228c7729684b3c830d5b6aa1626247, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/78dae72e30e94fe5ad4d4cd2c9164316, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ef6e007c369a4807962c04097d6da0da, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/91f4e6362c774f4fb32740ad8665b33c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/c0c07f51827e4578a10d8fd09b1c674b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ec7da7282a344cbd8b35d505fc5e81a6, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/b25db12053864b4b9346206c8db57594, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/db2a9ff078c7486aa825b22121e5cb1f, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/4f621884b8bb4b82a6ef440fffed828e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ac80965ac9094dd1a5d2a7b7815404e1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/7f5578842d064e618b351d7320e796a8, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/2312e656c7af42dd932156d2aa6e303a, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/8421c7c265044c09b8ccc6e0a53e646c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/f8a4358dcc6344f288d418fdfcf9d0cf, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/5117528cb1694a96a73b30df57583146, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/df94a3f1cfbc4c72b9a5076612a270e7] to archive 2024-11-14T21:35:04,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:35:04,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/545447f6859f4e83b3e13800a91b6ef2 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/545447f6859f4e83b3e13800a91b6ef2 2024-11-14T21:35:04,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/35da1178ea5445baa4787629c8a7aac1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/35da1178ea5445baa4787629c8a7aac1 2024-11-14T21:35:04,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ae1a92eb47c4452da72d7112a7eb2e30 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ae1a92eb47c4452da72d7112a7eb2e30 2024-11-14T21:35:04,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/1ad4548083d6477c8281289276e1a3e1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/1ad4548083d6477c8281289276e1a3e1 2024-11-14T21:35:04,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/9784e0d35d9c4ae4b533647328997a1d to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/9784e0d35d9c4ae4b533647328997a1d 2024-11-14T21:35:04,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/59eb90c1efca4d2e8cc347ccda32bbfb to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/59eb90c1efca4d2e8cc347ccda32bbfb 2024-11-14T21:35:04,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/54da3f03705f4caf82361fb01400953f to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/54da3f03705f4caf82361fb01400953f 2024-11-14T21:35:04,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/56228c7729684b3c830d5b6aa1626247 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/56228c7729684b3c830d5b6aa1626247 2024-11-14T21:35:04,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/78dae72e30e94fe5ad4d4cd2c9164316 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/78dae72e30e94fe5ad4d4cd2c9164316 2024-11-14T21:35:04,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ef6e007c369a4807962c04097d6da0da to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ef6e007c369a4807962c04097d6da0da 2024-11-14T21:35:04,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/91f4e6362c774f4fb32740ad8665b33c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/91f4e6362c774f4fb32740ad8665b33c 2024-11-14T21:35:04,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/c0c07f51827e4578a10d8fd09b1c674b to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/c0c07f51827e4578a10d8fd09b1c674b 2024-11-14T21:35:04,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ec7da7282a344cbd8b35d505fc5e81a6 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ec7da7282a344cbd8b35d505fc5e81a6 2024-11-14T21:35:04,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/b25db12053864b4b9346206c8db57594 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/b25db12053864b4b9346206c8db57594 2024-11-14T21:35:04,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/db2a9ff078c7486aa825b22121e5cb1f to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/db2a9ff078c7486aa825b22121e5cb1f 2024-11-14T21:35:04,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/4f621884b8bb4b82a6ef440fffed828e to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/4f621884b8bb4b82a6ef440fffed828e 2024-11-14T21:35:04,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ac80965ac9094dd1a5d2a7b7815404e1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/ac80965ac9094dd1a5d2a7b7815404e1 2024-11-14T21:35:04,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/7f5578842d064e618b351d7320e796a8 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/7f5578842d064e618b351d7320e796a8 2024-11-14T21:35:04,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/2312e656c7af42dd932156d2aa6e303a to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/2312e656c7af42dd932156d2aa6e303a 2024-11-14T21:35:04,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/8421c7c265044c09b8ccc6e0a53e646c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/8421c7c265044c09b8ccc6e0a53e646c 2024-11-14T21:35:04,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/f8a4358dcc6344f288d418fdfcf9d0cf to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/f8a4358dcc6344f288d418fdfcf9d0cf 2024-11-14T21:35:04,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/5117528cb1694a96a73b30df57583146 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/5117528cb1694a96a73b30df57583146 2024-11-14T21:35:04,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/df94a3f1cfbc4c72b9a5076612a270e7 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/df94a3f1cfbc4c72b9a5076612a270e7 2024-11-14T21:35:04,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/8aee2460f98d4b2db15151943146526e, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/58b389b9caa441e5b85675f839afdfdc, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/b98b82985aa74150b2a93aa08a7db1d2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/5f345f46158d4277a8c505dfaa2d445d, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/31fcefad9eab412ca066e6ed908f7abb, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/84d0c09ed321425ba8196374df4277c1, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/1b061f4858c2419a848f695e65b91816, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/05ae41184903409e8b9f066e25ae0917, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/dbdb2d0a597743e4b890e5a34c104fae, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/506e77835b954426a10ff71bdd169602, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/38f3375a7899472bbff0150d96b87074, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/13b759ca7f674c1b9fe356b4560d5a3c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/196ec192d74040e3b5c0c055086070db, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/70390e339293428c9f9b37fabf62219b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/195d24beeb64410189ef67706ce261c6, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/1fc7d0dc4c8046a29e90709f4bda6dc9, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/dc010a382f7d42df9d7d4242309f1039, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/8a9d12804525460eb69a2fbd5bbf512a, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/9da5ab10aa2e4fbdaf84e5cf07173f68, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/65c0e50197eb499e94e2f7e0d532dd12, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/a95a5f6a7bc348d89836d0ccbb2a407b, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/c423d3da07f64871802ffc75b808a643, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/2ec507b9e8814f9cb09942db95157597] to archive 2024-11-14T21:35:04,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:35:04,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/8aee2460f98d4b2db15151943146526e to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/8aee2460f98d4b2db15151943146526e 2024-11-14T21:35:04,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/58b389b9caa441e5b85675f839afdfdc to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/58b389b9caa441e5b85675f839afdfdc 2024-11-14T21:35:04,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/b98b82985aa74150b2a93aa08a7db1d2 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/b98b82985aa74150b2a93aa08a7db1d2 2024-11-14T21:35:04,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/5f345f46158d4277a8c505dfaa2d445d to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/5f345f46158d4277a8c505dfaa2d445d 2024-11-14T21:35:04,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/31fcefad9eab412ca066e6ed908f7abb to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/31fcefad9eab412ca066e6ed908f7abb 2024-11-14T21:35:04,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/84d0c09ed321425ba8196374df4277c1 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/84d0c09ed321425ba8196374df4277c1 2024-11-14T21:35:04,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/1b061f4858c2419a848f695e65b91816 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/1b061f4858c2419a848f695e65b91816 2024-11-14T21:35:04,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/05ae41184903409e8b9f066e25ae0917 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/05ae41184903409e8b9f066e25ae0917 2024-11-14T21:35:04,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/dbdb2d0a597743e4b890e5a34c104fae to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/dbdb2d0a597743e4b890e5a34c104fae 2024-11-14T21:35:04,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/506e77835b954426a10ff71bdd169602 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/506e77835b954426a10ff71bdd169602 2024-11-14T21:35:04,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/38f3375a7899472bbff0150d96b87074 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/38f3375a7899472bbff0150d96b87074 2024-11-14T21:35:04,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/13b759ca7f674c1b9fe356b4560d5a3c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/13b759ca7f674c1b9fe356b4560d5a3c 2024-11-14T21:35:04,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/196ec192d74040e3b5c0c055086070db to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/196ec192d74040e3b5c0c055086070db 2024-11-14T21:35:04,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/70390e339293428c9f9b37fabf62219b to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/70390e339293428c9f9b37fabf62219b 2024-11-14T21:35:04,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/195d24beeb64410189ef67706ce261c6 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/195d24beeb64410189ef67706ce261c6 2024-11-14T21:35:04,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/1fc7d0dc4c8046a29e90709f4bda6dc9 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/1fc7d0dc4c8046a29e90709f4bda6dc9 2024-11-14T21:35:04,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/dc010a382f7d42df9d7d4242309f1039 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/dc010a382f7d42df9d7d4242309f1039 2024-11-14T21:35:04,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/8a9d12804525460eb69a2fbd5bbf512a to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/8a9d12804525460eb69a2fbd5bbf512a 2024-11-14T21:35:04,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/9da5ab10aa2e4fbdaf84e5cf07173f68 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/9da5ab10aa2e4fbdaf84e5cf07173f68 2024-11-14T21:35:04,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/65c0e50197eb499e94e2f7e0d532dd12 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/65c0e50197eb499e94e2f7e0d532dd12 2024-11-14T21:35:04,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/a95a5f6a7bc348d89836d0ccbb2a407b to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/a95a5f6a7bc348d89836d0ccbb2a407b 2024-11-14T21:35:04,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/c423d3da07f64871802ffc75b808a643 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/c423d3da07f64871802ffc75b808a643 2024-11-14T21:35:04,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/2ec507b9e8814f9cb09942db95157597 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/2ec507b9e8814f9cb09942db95157597 2024-11-14T21:35:04,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/5bc6524593f54ff391e434133321505c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/f06fbadd06114f5ebb9109d04df94689, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/9279bf9db8154d4fa3bdbd927a81baf7, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/0099dad617d44601a267b6da352c5ec3, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/248b114d164445e990659a8cd6e968d2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d07f165926144a3dbe09d83e43f3dd61, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/e09380d449af400982f104cfea1580e6, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/c152f8a870484cfd9f00aca1ebd5042c, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/1dee0ab05eb84c87b866be5606917a59, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/84395575746244f598909d05c960eaea, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/36bd1486b9264d42accf5e0e2e369f98, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d27bf48fd1ea45a989c53a26e43e6d36, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/f9846e3f9b2745b4a07fa3ab21d451d5, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/244191d1444a47208f33ad9d2477d481, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/e636d7be966a401381e2c55f5cbb65e9, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/3e6df6fc519f4342aca75a6c0a0f59f2, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/cf7fec4542f24a1daac7ecbc8bbe5b35, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d50702ff4470463dac4498caea46f88a, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/a1d088ddf83547abb2c71aa6a7e6ac65, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/5a759fcd3d5f469b869eb4ff20a4ae42, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d1a2946c040641dfb9ea6ba6098cab95, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/6cf04d529a61429fa6542108479fb824, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/4438305d86fb4c79be7027b4f0056353] to archive 2024-11-14T21:35:04,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:35:04,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/5bc6524593f54ff391e434133321505c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/5bc6524593f54ff391e434133321505c 2024-11-14T21:35:04,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/f06fbadd06114f5ebb9109d04df94689 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/f06fbadd06114f5ebb9109d04df94689 2024-11-14T21:35:04,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/9279bf9db8154d4fa3bdbd927a81baf7 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/9279bf9db8154d4fa3bdbd927a81baf7 2024-11-14T21:35:04,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/0099dad617d44601a267b6da352c5ec3 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/0099dad617d44601a267b6da352c5ec3 2024-11-14T21:35:04,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/248b114d164445e990659a8cd6e968d2 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/248b114d164445e990659a8cd6e968d2 2024-11-14T21:35:04,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d07f165926144a3dbe09d83e43f3dd61 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d07f165926144a3dbe09d83e43f3dd61 2024-11-14T21:35:04,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/e09380d449af400982f104cfea1580e6 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/e09380d449af400982f104cfea1580e6 2024-11-14T21:35:04,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/c152f8a870484cfd9f00aca1ebd5042c to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/c152f8a870484cfd9f00aca1ebd5042c 2024-11-14T21:35:04,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/1dee0ab05eb84c87b866be5606917a59 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/1dee0ab05eb84c87b866be5606917a59 2024-11-14T21:35:04,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/84395575746244f598909d05c960eaea to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/84395575746244f598909d05c960eaea 2024-11-14T21:35:04,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/36bd1486b9264d42accf5e0e2e369f98 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/36bd1486b9264d42accf5e0e2e369f98 2024-11-14T21:35:04,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d27bf48fd1ea45a989c53a26e43e6d36 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d27bf48fd1ea45a989c53a26e43e6d36 2024-11-14T21:35:04,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/f9846e3f9b2745b4a07fa3ab21d451d5 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/f9846e3f9b2745b4a07fa3ab21d451d5 2024-11-14T21:35:04,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/244191d1444a47208f33ad9d2477d481 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/244191d1444a47208f33ad9d2477d481 2024-11-14T21:35:04,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/e636d7be966a401381e2c55f5cbb65e9 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/e636d7be966a401381e2c55f5cbb65e9 2024-11-14T21:35:04,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/3e6df6fc519f4342aca75a6c0a0f59f2 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/3e6df6fc519f4342aca75a6c0a0f59f2 2024-11-14T21:35:04,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/cf7fec4542f24a1daac7ecbc8bbe5b35 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/cf7fec4542f24a1daac7ecbc8bbe5b35 2024-11-14T21:35:04,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d50702ff4470463dac4498caea46f88a to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d50702ff4470463dac4498caea46f88a 2024-11-14T21:35:04,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/a1d088ddf83547abb2c71aa6a7e6ac65 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/a1d088ddf83547abb2c71aa6a7e6ac65 2024-11-14T21:35:04,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/5a759fcd3d5f469b869eb4ff20a4ae42 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/5a759fcd3d5f469b869eb4ff20a4ae42 2024-11-14T21:35:04,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d1a2946c040641dfb9ea6ba6098cab95 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d1a2946c040641dfb9ea6ba6098cab95 2024-11-14T21:35:04,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/6cf04d529a61429fa6542108479fb824 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/6cf04d529a61429fa6542108479fb824 2024-11-14T21:35:04,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/4438305d86fb4c79be7027b4f0056353 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/4438305d86fb4c79be7027b4f0056353 2024-11-14T21:35:04,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T21:35:04,892 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/recovered.edits/407.seqid, newMaxSeqId=407, maxSeqId=4 2024-11-14T21:35:04,893 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. 2024-11-14T21:35:04,893 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(1676): Region close journal for 2c62143a7911e8dec390d4d4558f8aa4: Waiting for close lock at 1731620102888Running coprocessor pre-close hooks at 1731620102888Disabling compacts and flushes for region at 1731620102888Disabling writes for close at 1731620103481 (+593 ms)Obtaining lock to block concurrent updates at 1731620103481Preparing flush snapshotting stores in 2c62143a7911e8dec390d4d4558f8aa4 at 1731620103481Finished memstore snapshotting TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731620103482 (+1 ms)Flushing stores of TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4. at 1731620103482Flushing 2c62143a7911e8dec390d4d4558f8aa4/A: creating writer at 1731620103482Flushing mob file 2c62143a7911e8dec390d4d4558f8aa4/A: appending metadata at 1731620103491 (+9 ms)Flushing mob file 2c62143a7911e8dec390d4d4558f8aa4/A: closing flushed file at 1731620103491Flushing 2c62143a7911e8dec390d4d4558f8aa4/A: appending metadata at 1731620103905 (+414 ms)Flushing 2c62143a7911e8dec390d4d4558f8aa4/A: closing flushed file at 1731620103905Flushing 2c62143a7911e8dec390d4d4558f8aa4/B: creating writer at 1731620104312 (+407 ms)Flushing 2c62143a7911e8dec390d4d4558f8aa4/B: appending metadata at 1731620104314 (+2 ms)Flushing 2c62143a7911e8dec390d4d4558f8aa4/B: closing flushed file at 1731620104314Flushing 2c62143a7911e8dec390d4d4558f8aa4/C: creating writer at 1731620104723 (+409 ms)Flushing 2c62143a7911e8dec390d4d4558f8aa4/C: appending metadata at 1731620104725 (+2 ms)Flushing 2c62143a7911e8dec390d4d4558f8aa4/C: closing flushed file at 1731620104725Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62e41492: reopening flushed file at 1731620104752 (+27 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c46b4bd: reopening flushed file at 1731620104756 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3de63262: reopening flushed file at 1731620104760 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 2c62143a7911e8dec390d4d4558f8aa4 in 1283ms, sequenceid=404, compaction requested=true at 1731620104765 (+5 ms)Writing region close event to WAL at 1731620104878 (+113 ms)Running coprocessor post-close hooks at 1731620104893 (+15 ms)Closed at 1731620104893 2024-11-14T21:35:04,896 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] handler.UnassignRegionHandler(157): Closed 2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:04,896 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=148 updating hbase:meta row=2c62143a7911e8dec390d4d4558f8aa4, regionState=CLOSED 2024-11-14T21:35:04,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 because future has completed 2024-11-14T21:35:04,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-14T21:35:04,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; CloseRegionProcedure 2c62143a7911e8dec390d4d4558f8aa4, server=ef788a4cade8,37683,1731619918560 in 2.1630 sec 2024-11-14T21:35:04,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-14T21:35:04,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c62143a7911e8dec390d4d4558f8aa4, UNASSIGN in 2.1690 sec 2024-11-14T21:35:04,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-14T21:35:04,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1740 sec 2024-11-14T21:35:04,908 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731620104908"}]},"ts":"1731620104908"} 2024-11-14T21:35:04,910 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T21:35:04,910 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T21:35:04,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1850 sec 2024-11-14T21:35:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T21:35:06,901 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:35:06,901 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T21:35:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:35:06,903 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=150, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:35:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T21:35:06,904 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=150, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:35:06,905 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,908 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A, FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B, FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C, FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/recovered.edits] 2024-11-14T21:35:06,910 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/0f59b33dbb5743468715192712a67728 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/0f59b33dbb5743468715192712a67728 2024-11-14T21:35:06,911 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/14685a1c507e433a9891da537d1e33da to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/14685a1c507e433a9891da537d1e33da 2024-11-14T21:35:06,912 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/1d116a861931444785ff6a49d143f5e7 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/1d116a861931444785ff6a49d143f5e7 2024-11-14T21:35:06,913 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/6414e4ba68854dbb8a6075aa095d7479 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/6414e4ba68854dbb8a6075aa095d7479 2024-11-14T21:35:06,914 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/734904b958db4e1c88adc37f68d30159 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/A/734904b958db4e1c88adc37f68d30159 2024-11-14T21:35:06,915 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/0fb1123b5e4a438299451cddf90391a6 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/0fb1123b5e4a438299451cddf90391a6 2024-11-14T21:35:06,916 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/51716121fc0c49f3b8dc16d1727bc290 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/51716121fc0c49f3b8dc16d1727bc290 2024-11-14T21:35:06,917 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/6e90a94828d14b1399bc9ba632e578a0 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/6e90a94828d14b1399bc9ba632e578a0 2024-11-14T21:35:06,918 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/75b9e3c4979d49beac8291a7edceee39 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/75b9e3c4979d49beac8291a7edceee39 2024-11-14T21:35:06,919 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/905e5723d52345939c83e9ac02916a58 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/B/905e5723d52345939c83e9ac02916a58 2024-11-14T21:35:06,920 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/3201afa336ca4fca9577ebcea5e7aeff to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/3201afa336ca4fca9577ebcea5e7aeff 2024-11-14T21:35:06,921 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/bf536083571243b4b5fc3e3c2231def4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/bf536083571243b4b5fc3e3c2231def4 2024-11-14T21:35:06,922 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d754475c722b40bf8fd3aad151a41ac2 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/d754475c722b40bf8fd3aad151a41ac2 2024-11-14T21:35:06,922 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/e72a04d3dccd4820b4932c235b33f48a to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/e72a04d3dccd4820b4932c235b33f48a 2024-11-14T21:35:06,923 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/f1e19067bed14dd28413071b656ed1f3 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/C/f1e19067bed14dd28413071b656ed1f3 2024-11-14T21:35:06,925 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/recovered.edits/407.seqid to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4/recovered.edits/407.seqid 2024-11-14T21:35:06,925 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/default/TestAcidGuarantees/2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,925 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T21:35:06,925 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T21:35:06,926 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T21:35:06,928 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111402dc5c014675412cb1af0683d3dbd920_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111402dc5c014675412cb1af0683d3dbd920_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,929 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114046c65f6dd42484f9b6243eeb70ef438_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114046c65f6dd42484f9b6243eeb70ef438_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,930 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114095805852a9548e59e4d9e585e80d0f6_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114095805852a9548e59e4d9e585e80d0f6_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,931 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111412793f684de1488caf64693f6cbe0e04_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111412793f684de1488caf64693f6cbe0e04_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,932 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141ac0ec7b998343b5a29ea6feb60e3245_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141ac0ec7b998343b5a29ea6feb60e3245_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,933 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142588a2b0782b4ac092e655ee11694ada_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142588a2b0782b4ac092e655ee11694ada_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,934 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143667a069443d4ef6a7da51be2a3672e4_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143667a069443d4ef6a7da51be2a3672e4_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,935 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111437c814af297f43a388f2d1ffb18c304b_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111437c814af297f43a388f2d1ffb18c304b_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,935 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145ff25110ffbb4b638cfc61e41803676e_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145ff25110ffbb4b638cfc61e41803676e_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,936 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111460c06ea1314b4a93b6f1f3bbd0b71d3e_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111460c06ea1314b4a93b6f1f3bbd0b71d3e_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,937 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111477d5304c62c1454daaa5d9351775abd9_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111477d5304c62c1454daaa5d9351775abd9_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,939 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111493de2fabf6164180aafbb45a2610df32_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111493de2fabf6164180aafbb45a2610df32_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,940 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149dcded3380a442efbfa136df243c4003_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149dcded3380a442efbfa136df243c4003_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,941 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114af1facec8e1b4eab89b2ce1e96093da4_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114af1facec8e1b4eab89b2ce1e96093da4_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,942 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b242676137b94d05b07e25af8d60afe0_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b242676137b94d05b07e25af8d60afe0_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,944 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b40818b523a84e1697ab00a009971b0f_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b40818b523a84e1697ab00a009971b0f_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,945 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bf4f9bd212744f8bba319e5898a8ca7e_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bf4f9bd212744f8bba319e5898a8ca7e_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,946 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f37a4cd9c3d5441abfe55568dc358c17_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f37a4cd9c3d5441abfe55568dc358c17_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,947 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f972713ada1c4094923073bad024abd9_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f972713ada1c4094923073bad024abd9_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,948 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fcbbc158b46d451ca7dc5c955700a3d9_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fcbbc158b46d451ca7dc5c955700a3d9_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,949 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fce9af9f0dfd460894648a515b4a8e49_2c62143a7911e8dec390d4d4558f8aa4 to hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fce9af9f0dfd460894648a515b4a8e49_2c62143a7911e8dec390d4d4558f8aa4 2024-11-14T21:35:06,950 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T21:35:06,952 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=150, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:35:06,954 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T21:35:06,956 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T21:35:06,957 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=150, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:35:06,957 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T21:35:06,957 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731620106957"}]},"ts":"9223372036854775807"} 2024-11-14T21:35:06,958 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T21:35:06,958 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2c62143a7911e8dec390d4d4558f8aa4, NAME => 'TestAcidGuarantees,,1731620078964.2c62143a7911e8dec390d4d4558f8aa4.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T21:35:06,958 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T21:35:06,959 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731620106958"}]},"ts":"9223372036854775807"} 2024-11-14T21:35:06,960 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T21:35:06,961 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=150, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T21:35:06,961 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 60 msec 2024-11-14T21:35:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T21:35:07,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T21:35:07,011 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T21:35:07,022 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 249) - Thread LEAK? -, OpenFileDescriptor=455 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=982 (was 1017), ProcessCount=11 (was 11), AvailableMemoryMB=1013 (was 777) - AvailableMemoryMB LEAK? - 2024-11-14T21:35:07,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T21:35:07,023 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T21:35:07,023 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T21:35:07,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:07,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:07,023 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:35:07,023 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T21:35:07,023 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1519452659, stopped=false 2024-11-14T21:35:07,023 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ef788a4cade8,38255,1731619917503 2024-11-14T21:35:07,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T21:35:07,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:35:07,026 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T21:35:07,027 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T21:35:07,027 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T21:35:07,027 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T21:35:07,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:07,028 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ef788a4cade8,37683,1731619918560' ***** 2024-11-14T21:35:07,028 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T21:35:07,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37683-0x100ac5a9af00001, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T21:35:07,028 INFO [RS:0;ef788a4cade8:37683 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T21:35:07,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37683-0x100ac5a9af00001, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:35:07,029 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T21:35:07,029 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37683-0x100ac5a9af00001, quorum=127.0.0.1:61142, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T21:35:07,029 INFO [RS:0;ef788a4cade8:37683 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T21:35:07,030 INFO [RS:0;ef788a4cade8:37683 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T21:35:07,030 INFO [RS:0;ef788a4cade8:37683 {}] regionserver.HRegionServer(959): stopping server ef788a4cade8,37683,1731619918560 2024-11-14T21:35:07,030 INFO [RS:0;ef788a4cade8:37683 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T21:35:07,030 INFO [RS:0;ef788a4cade8:37683 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ef788a4cade8:37683. 2024-11-14T21:35:07,030 DEBUG [RS:0;ef788a4cade8:37683 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T21:35:07,030 DEBUG [RS:0;ef788a4cade8:37683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:07,030 INFO [RS:0;ef788a4cade8:37683 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T21:35:07,030 INFO [RS:0;ef788a4cade8:37683 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T21:35:07,030 INFO [RS:0;ef788a4cade8:37683 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T21:35:07,030 INFO [RS:0;ef788a4cade8:37683 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T21:35:07,031 INFO [RS:0;ef788a4cade8:37683 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T21:35:07,031 DEBUG [RS:0;ef788a4cade8:37683 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-14T21:35:07,032 DEBUG [RS:0;ef788a4cade8:37683 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T21:35:07,032 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T21:35:07,032 INFO [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T21:35:07,032 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T21:35:07,032 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T21:35:07,032 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T21:35:07,032 INFO [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-14T21:35:07,065 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740/.tmp/info/cef777827973427c84fb5c2a52cdf1c9 is 95, key is TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa./info:/1731619956287/DeleteFamily/seqid=0 2024-11-14T21:35:07,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742488_1664 (size=6514) 2024-11-14T21:35:07,108 INFO [regionserver/ef788a4cade8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T21:35:07,129 INFO [regionserver/ef788a4cade8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T21:35:07,129 INFO [regionserver/ef788a4cade8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T21:35:07,232 DEBUG [RS:0;ef788a4cade8:37683 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T21:35:07,432 DEBUG [RS:0;ef788a4cade8:37683 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T21:35:07,470 INFO [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740/.tmp/info/cef777827973427c84fb5c2a52cdf1c9 2024-11-14T21:35:07,489 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740/.tmp/ns/9a49522f49774068b0aae74396be90ca is 93, key is TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa./ns:/1731619956231/DeleteFamily/seqid=0 2024-11-14T21:35:07,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742489_1665 (size=5906) 2024-11-14T21:35:07,632 DEBUG [RS:0;ef788a4cade8:37683 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T21:35:07,832 DEBUG [RS:0;ef788a4cade8:37683 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T21:35:07,895 INFO [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740/.tmp/ns/9a49522f49774068b0aae74396be90ca 2024-11-14T21:35:07,940 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740/.tmp/rep_barrier/d464a4a95e0246f4b229e807be4223df is 102, key is TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa./rep_barrier:/1731619956231/DeleteFamily/seqid=0 2024-11-14T21:35:07,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742490_1666 (size=6025) 2024-11-14T21:35:08,033 DEBUG [RS:0;ef788a4cade8:37683 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T21:35:08,077 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T21:35:08,233 DEBUG [RS:0;ef788a4cade8:37683 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T21:35:08,349 INFO [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740/.tmp/rep_barrier/d464a4a95e0246f4b229e807be4223df 2024-11-14T21:35:08,376 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740/.tmp/table/3918ab8491174f91921ca10612b98dec is 96, key is TestAcidGuarantees,,1731619922287.e9722fc966803fe6d499d80c4851a0fa./table:/1731619956231/DeleteFamily/seqid=0 2024-11-14T21:35:08,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742491_1667 (size=5936) 2024-11-14T21:35:08,386 INFO [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740/.tmp/table/3918ab8491174f91921ca10612b98dec 2024-11-14T21:35:08,390 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740/.tmp/info/cef777827973427c84fb5c2a52cdf1c9 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740/info/cef777827973427c84fb5c2a52cdf1c9 2024-11-14T21:35:08,395 INFO [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740/info/cef777827973427c84fb5c2a52cdf1c9, entries=12, sequenceid=90, filesize=6.4 K 2024-11-14T21:35:08,396 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740/.tmp/ns/9a49522f49774068b0aae74396be90ca as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740/ns/9a49522f49774068b0aae74396be90ca 2024-11-14T21:35:08,400 INFO [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740/ns/9a49522f49774068b0aae74396be90ca, entries=8, sequenceid=90, filesize=5.8 K 2024-11-14T21:35:08,400 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740/.tmp/rep_barrier/d464a4a95e0246f4b229e807be4223df as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740/rep_barrier/d464a4a95e0246f4b229e807be4223df 2024-11-14T21:35:08,404 INFO [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740/rep_barrier/d464a4a95e0246f4b229e807be4223df, entries=6, sequenceid=90, filesize=5.9 K 2024-11-14T21:35:08,405 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740/.tmp/table/3918ab8491174f91921ca10612b98dec as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740/table/3918ab8491174f91921ca10612b98dec 2024-11-14T21:35:08,408 INFO [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740/table/3918ab8491174f91921ca10612b98dec, entries=7, sequenceid=90, filesize=5.8 K 2024-11-14T21:35:08,409 INFO [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1376ms, sequenceid=90, compaction requested=false 2024-11-14T21:35:08,415 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-14T21:35:08,415 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T21:35:08,416 INFO [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T21:35:08,416 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731620107031Running coprocessor pre-close hooks at 1731620107031Disabling compacts and flushes for region at 1731620107031Disabling writes for close at 1731620107032 (+1 ms)Obtaining lock to block concurrent updates at 1731620107032Preparing flush snapshotting stores in 1588230740 at 1731620107032Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731620107033 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731620107033Flushing 1588230740/info: creating writer at 1731620107034 (+1 ms)Flushing 1588230740/info: appending metadata at 1731620107064 (+30 ms)Flushing 1588230740/info: closing flushed file at 1731620107064Flushing 1588230740/ns: creating writer at 1731620107474 (+410 ms)Flushing 1588230740/ns: appending metadata at 1731620107489 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731620107489Flushing 1588230740/rep_barrier: creating writer at 1731620107905 (+416 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731620107940 (+35 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731620107940Flushing 1588230740/table: creating writer at 1731620108353 (+413 ms)Flushing 1588230740/table: appending metadata at 1731620108375 (+22 ms)Flushing 1588230740/table: closing flushed file at 1731620108375Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fefeb86: reopening flushed file at 1731620108390 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fa0a515: reopening flushed file at 1731620108395 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ff20eff: reopening flushed file at 1731620108400 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9764ef5: reopening flushed file at 1731620108404 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1376ms, sequenceid=90, compaction requested=false at 1731620108409 (+5 ms)Writing region close event to WAL at 1731620108412 (+3 ms)Running coprocessor post-close hooks at 1731620108415 (+3 ms)Closed at 1731620108416 (+1 ms) 2024-11-14T21:35:08,416 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T21:35:08,433 INFO [RS:0;ef788a4cade8:37683 {}] regionserver.HRegionServer(976): stopping server ef788a4cade8,37683,1731619918560; all regions closed. 2024-11-14T21:35:08,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741834_1010 (size=25344) 2024-11-14T21:35:08,445 DEBUG [RS:0;ef788a4cade8:37683 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/oldWALs 2024-11-14T21:35:08,445 INFO [RS:0;ef788a4cade8:37683 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef788a4cade8%2C37683%2C1731619918560.meta:.meta(num 1731619921796) 2024-11-14T21:35:08,454 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/WALs/ef788a4cade8,37683,1731619918560/ef788a4cade8%2C37683%2C1731619918560.1731619921321 not finished, retry = 0 2024-11-14T21:35:08,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741833_1009 (size=18144236) 2024-11-14T21:35:08,557 DEBUG [RS:0;ef788a4cade8:37683 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/oldWALs 2024-11-14T21:35:08,557 INFO [RS:0;ef788a4cade8:37683 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef788a4cade8%2C37683%2C1731619918560:(num 1731619921321) 2024-11-14T21:35:08,557 DEBUG [RS:0;ef788a4cade8:37683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:35:08,557 INFO [RS:0;ef788a4cade8:37683 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T21:35:08,557 INFO [RS:0;ef788a4cade8:37683 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T21:35:08,557 INFO [RS:0;ef788a4cade8:37683 {}] hbase.ChoreService(370): Chore service for: regionserver/ef788a4cade8:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T21:35:08,557 INFO [regionserver/ef788a4cade8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T21:35:08,558 INFO [RS:0;ef788a4cade8:37683 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T21:35:08,559 INFO [RS:0;ef788a4cade8:37683 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37683 2024-11-14T21:35:08,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37683-0x100ac5a9af00001, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ef788a4cade8,37683,1731619918560 2024-11-14T21:35:08,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T21:35:08,564 INFO [RS:0;ef788a4cade8:37683 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T21:35:08,565 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ef788a4cade8,37683,1731619918560] 2024-11-14T21:35:08,567 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ef788a4cade8,37683,1731619918560 already deleted, retry=false 2024-11-14T21:35:08,567 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ef788a4cade8,37683,1731619918560 expired; onlineServers=0 2024-11-14T21:35:08,568 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ef788a4cade8,38255,1731619917503' ***** 2024-11-14T21:35:08,568 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T21:35:08,568 INFO [M:0;ef788a4cade8:38255 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T21:35:08,568 INFO [M:0;ef788a4cade8:38255 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T21:35:08,568 DEBUG [M:0;ef788a4cade8:38255 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T21:35:08,568 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T21:35:08,568 DEBUG [M:0;ef788a4cade8:38255 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T21:35:08,568 DEBUG [master/ef788a4cade8:0:becomeActiveMaster-HFileCleaner.large.0-1731619920977 {}] cleaner.HFileCleaner(306): Exit Thread[master/ef788a4cade8:0:becomeActiveMaster-HFileCleaner.large.0-1731619920977,5,FailOnTimeoutGroup] 2024-11-14T21:35:08,568 INFO [M:0;ef788a4cade8:38255 {}] hbase.ChoreService(370): Chore service for: master/ef788a4cade8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T21:35:08,568 INFO [M:0;ef788a4cade8:38255 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T21:35:08,568 DEBUG [master/ef788a4cade8:0:becomeActiveMaster-HFileCleaner.small.0-1731619920981 {}] cleaner.HFileCleaner(306): Exit Thread[master/ef788a4cade8:0:becomeActiveMaster-HFileCleaner.small.0-1731619920981,5,FailOnTimeoutGroup] 2024-11-14T21:35:08,568 DEBUG [M:0;ef788a4cade8:38255 {}] master.HMaster(1795): Stopping service threads 2024-11-14T21:35:08,569 INFO [M:0;ef788a4cade8:38255 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T21:35:08,569 INFO [M:0;ef788a4cade8:38255 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T21:35:08,569 ERROR [M:0;ef788a4cade8:38255 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-14T21:35:08,570 INFO [M:0;ef788a4cade8:38255 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T21:35:08,570 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T21:35:08,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T21:35:08,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:35:08,572 DEBUG [M:0;ef788a4cade8:38255 {}] zookeeper.ZKUtil(347): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T21:35:08,572 WARN [M:0;ef788a4cade8:38255 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T21:35:08,572 INFO [M:0;ef788a4cade8:38255 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/.lastflushedseqids 2024-11-14T21:35:08,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742492_1668 (size=72) 2024-11-14T21:35:08,668 INFO [RS:0;ef788a4cade8:37683 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T21:35:08,668 INFO [RS:0;ef788a4cade8:37683 {}] regionserver.HRegionServer(1031): Exiting; stopping=ef788a4cade8,37683,1731619918560; zookeeper connection closed. 2024-11-14T21:35:08,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37683-0x100ac5a9af00001, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T21:35:08,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37683-0x100ac5a9af00001, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T21:35:08,669 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@adbf66b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@adbf66b 2024-11-14T21:35:08,669 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-14T21:35:09,011 INFO [M:0;ef788a4cade8:38255 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T21:35:09,012 INFO [M:0;ef788a4cade8:38255 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T21:35:09,012 DEBUG [M:0;ef788a4cade8:38255 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T21:35:09,012 INFO [M:0;ef788a4cade8:38255 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T21:35:09,012 DEBUG [M:0;ef788a4cade8:38255 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T21:35:09,012 DEBUG [M:0;ef788a4cade8:38255 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T21:35:09,012 DEBUG [M:0;ef788a4cade8:38255 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T21:35:09,012 INFO [M:0;ef788a4cade8:38255 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=776.24 KB heapSize=951.84 KB 2024-11-14T21:35:09,030 DEBUG [M:0;ef788a4cade8:38255 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7275cda1406841ea96da71d62b340265 is 82, key is hbase:meta,,1/info:regioninfo/1731619921897/Put/seqid=0 2024-11-14T21:35:09,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742493_1669 (size=5672) 2024-11-14T21:35:09,435 INFO [M:0;ef788a4cade8:38255 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2188 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7275cda1406841ea96da71d62b340265 2024-11-14T21:35:09,466 DEBUG [M:0;ef788a4cade8:38255 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7c739d39eb9149ad827120d648062851 is 2479, key is \x00\x00\x00\x00\x00\x00\x00\x7F/proc:d/1731620082073/Put/seqid=0 2024-11-14T21:35:09,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742494_1670 (size=41869) 2024-11-14T21:35:09,475 INFO [M:0;ef788a4cade8:38255 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=775.69 KB at sequenceid=2188 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7c739d39eb9149ad827120d648062851 2024-11-14T21:35:09,494 INFO [M:0;ef788a4cade8:38255 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7c739d39eb9149ad827120d648062851 2024-11-14T21:35:09,516 DEBUG [M:0;ef788a4cade8:38255 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c335ac56a1534a1f83196d56f18a484a is 69, key is ef788a4cade8,37683,1731619918560/rs:state/1731619921010/Put/seqid=0 2024-11-14T21:35:09,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073742495_1671 (size=5156) 2024-11-14T21:35:09,929 INFO [M:0;ef788a4cade8:38255 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2188 (bloomFilter=true), to=hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c335ac56a1534a1f83196d56f18a484a 2024-11-14T21:35:09,934 DEBUG [M:0;ef788a4cade8:38255 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7275cda1406841ea96da71d62b340265 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7275cda1406841ea96da71d62b340265 2024-11-14T21:35:09,938 INFO [M:0;ef788a4cade8:38255 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7275cda1406841ea96da71d62b340265, entries=8, sequenceid=2188, filesize=5.5 K 2024-11-14T21:35:09,939 DEBUG [M:0;ef788a4cade8:38255 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7c739d39eb9149ad827120d648062851 as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7c739d39eb9149ad827120d648062851 2024-11-14T21:35:09,943 INFO [M:0;ef788a4cade8:38255 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7c739d39eb9149ad827120d648062851 2024-11-14T21:35:09,943 INFO [M:0;ef788a4cade8:38255 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7c739d39eb9149ad827120d648062851, entries=150, sequenceid=2188, filesize=40.9 K 2024-11-14T21:35:09,943 DEBUG [M:0;ef788a4cade8:38255 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c335ac56a1534a1f83196d56f18a484a as hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c335ac56a1534a1f83196d56f18a484a 2024-11-14T21:35:09,947 INFO [M:0;ef788a4cade8:38255 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42139/user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c335ac56a1534a1f83196d56f18a484a, entries=1, sequenceid=2188, filesize=5.0 K 2024-11-14T21:35:09,948 INFO [M:0;ef788a4cade8:38255 {}] regionserver.HRegion(3140): Finished flush of dataSize ~776.24 KB/794871, heapSize ~951.55 KB/974384, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 936ms, sequenceid=2188, compaction requested=false 2024-11-14T21:35:09,954 INFO [M:0;ef788a4cade8:38255 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T21:35:09,954 DEBUG [M:0;ef788a4cade8:38255 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731620109012Disabling compacts and flushes for region at 1731620109012Disabling writes for close at 1731620109012Obtaining lock to block concurrent updates at 1731620109012Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731620109012Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=794871, getHeapSize=974624, getOffHeapSize=0, getCellsCount=2363 at 1731620109013 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731620109013Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731620109013Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731620109030 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731620109030Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731620109440 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731620109465 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731620109465Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731620109494 (+29 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731620109516 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731620109516Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a0d6863: reopening flushed file at 1731620109933 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2deae59e: reopening flushed file at 1731620109938 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bd79154: reopening flushed file at 1731620109943 (+5 ms)Finished flush of dataSize ~776.24 KB/794871, heapSize ~951.55 KB/974384, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 936ms, sequenceid=2188, compaction requested=false at 1731620109948 (+5 ms)Writing region close event to WAL at 1731620109954 (+6 ms)Closed at 1731620109954 2024-11-14T21:35:09,957 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/b9b6a972-c2c9-4d40-854f-cb1f9367f275/MasterData/WALs/ef788a4cade8,38255,1731619917503/ef788a4cade8%2C38255%2C1731619917503.1731619920124 not finished, retry = 0 2024-11-14T21:35:09,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40829 is added to blk_1073741830_1006 (size=938210) 2024-11-14T21:35:10,058 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T21:35:10,059 INFO [M:0;ef788a4cade8:38255 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T21:35:10,059 INFO [M:0;ef788a4cade8:38255 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38255 2024-11-14T21:35:10,059 INFO [M:0;ef788a4cade8:38255 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T21:35:10,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T21:35:10,163 INFO [M:0;ef788a4cade8:38255 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T21:35:10,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38255-0x100ac5a9af00000, quorum=127.0.0.1:61142, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T21:35:10,172 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@75b88559{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T21:35:10,175 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6e13e45b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T21:35:10,175 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T21:35:10,175 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72c5887c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T21:35:10,176 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@554622cc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/hadoop.log.dir/,STOPPED} 2024-11-14T21:35:10,181 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T21:35:10,181 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T21:35:10,181 WARN [BP-893082529-172.17.0.2-1731619913628 heartbeating to localhost/127.0.0.1:42139 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T21:35:10,181 WARN [BP-893082529-172.17.0.2-1731619913628 heartbeating to localhost/127.0.0.1:42139 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-893082529-172.17.0.2-1731619913628 (Datanode Uuid 5ce7ca41-404e-436f-a55b-ea22d766e048) service to localhost/127.0.0.1:42139 2024-11-14T21:35:10,185 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/cluster_a38eb657-3cf0-cdbb-aff3-937f25906809/data/data1/current/BP-893082529-172.17.0.2-1731619913628 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T21:35:10,185 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/cluster_a38eb657-3cf0-cdbb-aff3-937f25906809/data/data2/current/BP-893082529-172.17.0.2-1731619913628 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T21:35:10,186 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T21:35:10,219 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3654837{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T21:35:10,223 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3edc1d8c{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T21:35:10,223 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T21:35:10,223 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a4bbe01{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T21:35:10,223 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@493a7020{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4f0d9638-b47f-0e91-ec17-69c7a3eb01f5/hadoop.log.dir/,STOPPED} 2024-11-14T21:35:10,251 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T21:35:10,414 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T21:35:10,417 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T21:35:10,439 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down